forked from hans/Nominatim
automatic migration from 3.6 release
Adds a 'admin --migrate' command that checks for the current database version and runs any necessary migrations. Also has migrations going back to 3.6.
This commit is contained in:
Sarah Hoffmann
@@ -20,13 +20,16 @@ class AdminFuncs:
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
group = parser.add_argument_group('Admin task arguments')
|
||||
group.add_argument('--warm', action='store_true',
|
||||
help='Warm database caches for search and reverse queries.')
|
||||
group.add_argument('--check-database', action='store_true',
|
||||
help='Check that the database is complete and operational.')
|
||||
group.add_argument('--analyse-indexing', action='store_true',
|
||||
help='Print performance analysis of the indexing process.')
|
||||
group = parser.add_argument_group('Admin tasks')
|
||||
objs = group.add_mutually_exclusive_group(required=True)
|
||||
objs.add_argument('--warm', action='store_true',
|
||||
help='Warm database caches for search and reverse queries.')
|
||||
objs.add_argument('--check-database', action='store_true',
|
||||
help='Check that the database is complete and operational.')
|
||||
objs.add_argument('--migrate', action='store_true',
|
||||
help='Migrate the database to a new software version.')
|
||||
objs.add_argument('--analyse-indexing', action='store_true',
|
||||
help='Print performance analysis of the indexing process.')
|
||||
group = parser.add_argument_group('Arguments for cache warming')
|
||||
group.add_argument('--search-only', action='store_const', dest='target',
|
||||
const='search',
|
||||
@@ -44,7 +47,7 @@ class AdminFuncs:
|
||||
@staticmethod
|
||||
def run(args):
|
||||
if args.warm:
|
||||
AdminFuncs._warm(args)
|
||||
return AdminFuncs._warm(args)
|
||||
|
||||
if args.check_database:
|
||||
LOG.warning('Checking database')
|
||||
@@ -56,8 +59,14 @@ class AdminFuncs:
|
||||
from ..tools import admin
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id)
|
||||
return 0
|
||||
|
||||
return 0
|
||||
if args.migrate:
|
||||
LOG.warning('Checking for necessary database migrations')
|
||||
from ..tools import migration
|
||||
return migration.migrate(args.config, args)
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
@staticmethod
|
||||
|
||||
132
nominatim/tools/migration.py
Normal file
132
nominatim/tools/migration.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""
|
||||
Functions for database migration to newer software versions.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from ..db import properties
|
||||
from ..db.connection import connect
|
||||
from ..version import NOMINATIM_VERSION
|
||||
from . import refresh, database_import
|
||||
from ..errors import UsageError
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
_MIGRATION_FUNCTIONS = []
|
||||
|
||||
def migrate(config, paths):
|
||||
""" Check for the current database version and execute migrations,
|
||||
if necesssary.
|
||||
"""
|
||||
with connect(config.get_libpq_dsn()) as conn:
|
||||
if conn.table_exists('nominatim_properties'):
|
||||
db_version_str = properties.get_property(conn, 'database_version')
|
||||
else:
|
||||
db_version_str = None
|
||||
|
||||
if db_version_str is not None:
|
||||
parts = db_version_str.split('.')
|
||||
db_version = tuple([int(x) for x in parts[:2] + parts[2].split('-')])
|
||||
|
||||
if db_version == NOMINATIM_VERSION:
|
||||
LOG.warning("Database already at latest version (%s)", db_version_str)
|
||||
return 0
|
||||
|
||||
LOG.info("Detected database version: %s", db_version_str)
|
||||
else:
|
||||
db_version = _guess_version(conn)
|
||||
|
||||
|
||||
has_run_migration = False
|
||||
for version, func in _MIGRATION_FUNCTIONS:
|
||||
if db_version <= version:
|
||||
LOG.warning("Runnning: %s (%s)", func.__doc__.split('\n', 1)[0],
|
||||
'{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(version))
|
||||
kwargs = dict(conn=conn, config=config, paths=paths)
|
||||
func(**kwargs)
|
||||
has_run_migration = True
|
||||
|
||||
if has_run_migration:
|
||||
LOG.warning('Updating SQL functions.')
|
||||
refresh.create_functions(conn, config, paths.sqllib_dir)
|
||||
|
||||
properties.set_property(conn, 'database_version',
|
||||
'{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))
|
||||
|
||||
conn.commit()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _guess_version(conn):
|
||||
""" Guess a database version when there is no property table yet.
|
||||
Only migrations for 3.6 and later are supported, so bail out
|
||||
when the version seems older.
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
# In version 3.6, the country_name table was updated. Check for that.
|
||||
cnt = cur.scalar("""SELECT count(*) FROM
|
||||
(SELECT svals(name) FROM country_name
|
||||
WHERE country_code = 'gb')x;
|
||||
""")
|
||||
if cnt < 100:
|
||||
LOG.fatal('It looks like your database was imported with a version '
|
||||
'prior to 3.6.0. Automatic migration not possible.')
|
||||
raise UsageError('Migration not possible.')
|
||||
|
||||
return (3, 5, 0, 99)
|
||||
|
||||
|
||||
|
||||
def _migration(major, minor, patch=0, dbpatch=0):
|
||||
""" Decorator for a single migration step. The parameters describe the
|
||||
version after which the migration is applicable, i.e before changing
|
||||
from the given version to the next, the migration is required.
|
||||
|
||||
All migrations are run in the order in which they are defined in this
|
||||
file. Do not run global SQL scripts for migrations as you cannot be sure
|
||||
that these scripts do the same in later versions.
|
||||
|
||||
Functions will always be reimported in full at the end of the migration
|
||||
process, so the migration functions may leave a temporary state behind
|
||||
there.
|
||||
"""
|
||||
def decorator(func):
|
||||
_MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
@_migration(3, 5, 0, 99)
|
||||
def import_status_timestamp_change(conn, **_):
|
||||
""" Add timezone to timestamp in status table.
|
||||
|
||||
The import_status table has been changed to include timezone information
|
||||
with the time stamp.
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""ALTER TABLE import_status ALTER COLUMN lastimportdate
|
||||
TYPE timestamp with time zone;""")
|
||||
|
||||
|
||||
@_migration(3, 5, 0, 99)
|
||||
def install_database_module_in_project_directory(conn, config, paths, **_):
|
||||
""" Install database module in project directory.
|
||||
|
||||
The database module needs to be present in the project directory
|
||||
since those were introduced.
|
||||
"""
|
||||
database_import.install_module(paths.module_dir, paths.project_dir,
|
||||
config.DATABASE_MODULE_PATH, conn=conn)
|
||||
|
||||
|
||||
@_migration(3, 5, 0, 99)
|
||||
def add_nominatim_property_table(conn, config, **_):
|
||||
""" Add nominatim_property table.
|
||||
"""
|
||||
if not conn.table_exists('nominatim_properties'):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""CREATE TABLE nominatim_properties (
|
||||
property TEXT,
|
||||
value TEXT);
|
||||
GRANT SELECT ON TABLE nominatim_properties TO "{}";
|
||||
""".format(config.DATABASE_WEBUSER))
|
||||
Reference in New Issue
Block a user