automatic migration from 3.6 release

Adds a 'admin --migrate' command that checks for the current
database version and runs any necessary migrations. Also
has migrations going back to 3.6.

nominatim/clicmd/admin.py

@@ -20,13 +20,16 @@ class AdminFuncs:
 
     @staticmethod
     def add_args(parser):
-        group = parser.add_argument_group('Admin task arguments')
+        group = parser.add_argument_group('Admin tasks')
-        group.add_argument('--warm', action='store_true',
+        objs = group.add_mutually_exclusive_group(required=True)
-                           help='Warm database caches for search and reverse queries.')
+        objs.add_argument('--warm', action='store_true',
-        group.add_argument('--check-database', action='store_true',
+                          help='Warm database caches for search and reverse queries.')
-                           help='Check that the database is complete and operational.')
+        objs.add_argument('--check-database', action='store_true',
-        group.add_argument('--analyse-indexing', action='store_true',
+                          help='Check that the database is complete and operational.')
-                           help='Print performance analysis of the indexing process.')
+        objs.add_argument('--migrate', action='store_true',
+                          help='Migrate the database to a new software version.')
+        objs.add_argument('--analyse-indexing', action='store_true',
+                          help='Print performance analysis of the indexing process.')
         group = parser.add_argument_group('Arguments for cache warming')
         group.add_argument('--search-only', action='store_const', dest='target',
                            const='search',
@@ -44,7 +47,7 @@ class AdminFuncs:
     @staticmethod
     def run(args):
         if args.warm:
-            AdminFuncs._warm(args)
+            return AdminFuncs._warm(args)
 
         if args.check_database:
             LOG.warning('Checking database')
@@ -56,8 +59,14 @@ class AdminFuncs:
             from ..tools import admin
             with connect(args.config.get_libpq_dsn()) as conn:
                 admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id)
+            return 0
 
-        return 0
+        if args.migrate:
+            LOG.warning('Checking for necessary database migrations')
+            from ..tools import migration
+            return migration.migrate(args.config, args)
+
+        return 1
 
 
     @staticmethod

nominatim/tools/migration.py

@@ -0,0 +1,132 @@
+"""
+Functions for database migration to newer software versions.
+"""
+import logging
+
+from ..db import properties
+from ..db.connection import connect
+from ..version import NOMINATIM_VERSION
+from . import refresh, database_import
+from ..errors import UsageError
+
+LOG = logging.getLogger()
+
+_MIGRATION_FUNCTIONS = []
+
+def migrate(config, paths):
+    """ Check for the current database version and execute migrations,
+        if necesssary.
+    """
+    with connect(config.get_libpq_dsn()) as conn:
+        if conn.table_exists('nominatim_properties'):
+            db_version_str = properties.get_property(conn, 'database_version')
+        else:
+            db_version_str = None
+
+        if db_version_str is not None:
+            parts = db_version_str.split('.')
+            db_version = tuple([int(x) for x in parts[:2] + parts[2].split('-')])
+
+            if db_version == NOMINATIM_VERSION:
+                LOG.warning("Database already at latest version (%s)", db_version_str)
+                return 0
+
+            LOG.info("Detected database version: %s", db_version_str)
+        else:
+            db_version = _guess_version(conn)
+
+
+        has_run_migration = False
+        for version, func in _MIGRATION_FUNCTIONS:
+            if db_version <= version:
+                LOG.warning("Runnning: %s (%s)", func.__doc__.split('\n', 1)[0],
+                            '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(version))
+                kwargs = dict(conn=conn, config=config, paths=paths)
+                func(**kwargs)
+                has_run_migration = True
+
+        if has_run_migration:
+            LOG.warning('Updating SQL functions.')
+            refresh.create_functions(conn, config, paths.sqllib_dir)
+
+        properties.set_property(conn, 'database_version',
+                                '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))
+
+        conn.commit()
+
+    return 0
+
+
+def _guess_version(conn):
+    """ Guess a database version when there is no property table yet.
+        Only migrations for 3.6 and later are supported, so bail out
+        when the version seems older.
+    """
+    with conn.cursor() as cur:
+        # In version 3.6, the country_name table was updated. Check for that.
+        cnt = cur.scalar("""SELECT count(*) FROM
+                            (SELECT svals(name) FROM  country_name
+                             WHERE country_code = 'gb')x;
+                         """)
+        if cnt < 100:
+            LOG.fatal('It looks like your database was imported with a version '
+                      'prior to 3.6.0. Automatic migration not possible.')
+            raise UsageError('Migration not possible.')
+
+    return (3, 5, 0, 99)
+
+
+
+def _migration(major, minor, patch=0, dbpatch=0):
+    """ Decorator for a single migration step. The parameters describe the
+        version after which the migration is applicable, i.e before changing
+        from the given version to the next, the migration is required.
+
+        All migrations are run in the order in which they are defined in this
+        file. Do not run global SQL scripts for migrations as you cannot be sure
+        that these scripts do the same in later versions.
+
+        Functions will always be reimported in full at the end of the migration
+        process, so the migration functions may leave a temporary state behind
+        there.
+    """
+    def decorator(func):
+        _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
+
+    return decorator
+
+
+@_migration(3, 5, 0, 99)
+def import_status_timestamp_change(conn, **_):
+    """ Add timezone to timestamp in status table.
+
+        The import_status table has been changed to include timezone information
+        with the time stamp.
+    """
+    with conn.cursor() as cur:
+        cur.execute("""ALTER TABLE import_status ALTER COLUMN lastimportdate
+                       TYPE timestamp with time zone;""")
+
+
+@_migration(3, 5, 0, 99)
+def install_database_module_in_project_directory(conn, config, paths, **_):
+    """ Install database module in project directory.
+
+        The database module needs to be present in the project directory
+        since those were introduced.
+    """
+    database_import.install_module(paths.module_dir, paths.project_dir,
+                                   config.DATABASE_MODULE_PATH, conn=conn)
+
+
+@_migration(3, 5, 0, 99)
+def add_nominatim_property_table(conn, config, **_):
+    """ Add nominatim_property table.
+    """
+    if not conn.table_exists('nominatim_properties'):
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE nominatim_properties (
+                               property TEXT,
+                               value TEXT);
+                           GRANT SELECT ON TABLE nominatim_properties TO "{}";
+                        """.format(config.DATABASE_WEBUSER))