hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-12 22:04:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial implementation of GeoTIFF import functionality

Browse Source
This commit is contained in:
Tareq Al-Ahdal
2022-07-06 08:16:41 +02:00
committed by Sarah Hoffmann
parent 3381a92d92
commit c85b74497b
9 changed files with 78 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
nominatim/clicmd/refresh.py
View File

@@ -63,6 +63,8 @@ class UpdateRefresh:
help='Update the PL/pgSQL functions in the database')
group.add_argument('--wiki-data', action='store_true',
help='Update Wikipedia/data importance numbers')
group.add_argument('--osm-views', action='store_true',
help='Update OSM views/data importance numbers')
group.add_argument('--importance', action='store_true',
help='Recompute place importances (expensive!)')
group.add_argument('--website', action='store_true',
@@ -131,6 +133,15 @@ class UpdateRefresh:
LOG.fatal('FATAL: Wikipedia importance dump file not found')
return 1
if args.osm_views:
data_path = Path(args.config.OSM_VIEWS_DATA_PATH
or args.project_dir)
LOG.warning('Import OSM views GeoTIFF data from %s', data_path)
if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(),
data_path) > 0:
LOG.fatal('FATAL: OSM views GeoTIFF file not found')
return 1
# Attention: importance MUST come after wiki data import.
if args.importance:
LOG.warning('Update importance values for database')

14
nominatim/clicmd/setup.py
View File

@@ -58,8 +58,10 @@ class SetupAll:
group2.add_argument('--no-updates', action='store_true',
help="Do not keep tables that are only needed for "
"updating the database later")
group2.add_argument('--osm-views', action='store_true',
help='Import OSM views GeoTIFF')
group2.add_argument('--offline', action='store_true',
help="Do not attempt to load any additional data from the internet")
help="Do not attempt to load any additional data from the internet")
group3 = parser.add_argument_group('Expert options')
group3.add_argument('--ignore-errors', action='store_true',
help='Continue import even when errors in SQL are present')
@@ -103,7 +105,15 @@ class SetupAll:
if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
data_path) > 0:
LOG.error('Wikipedia importance dump file not found. '
'Will be using default importances.')
'Calculating importance values of locations will not use Wikipedia importance data.')
LOG.warning('Importing OSM views GeoTIFF data')
database_import.import_osm_views_geotiff()
data_path = Path(args.config.OSM_VIEWS_DATA_PATH or args.project_dir)
if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(),
data_path) > 0:
LOG.error('OSM views GeoTIFF file not found. '
'Calculating importance values of locations will not use OSM views data.')
if args.continue_at is None or args.continue_at == 'load-data':
LOG.warning('Initialise tables')

6
nominatim/tools/database_import.py
View File

@@ -75,6 +75,7 @@ def setup_database_skeleton(dsn: str, rouser: Optional[str] = None) -> None:
with conn.cursor() as cur:
cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')
conn.commit()
_require_version('PostGIS',
@@ -246,3 +247,8 @@ def create_search_indices(conn: Connection, config: Configuration,
sql.run_parallel_sql_file(config.get_libpq_dsn(),
'indices.sql', min(8, threads), drop=drop)
def import_osm_views_geotiff():
"""Import OSM views GeoTIFF file"""
subprocess.run("raster2pgsql -s 4326 -I -C -t 100x100 -e osmviews.tiff public.osmviews | psql nominatim", shell=True, check=True)

19
nominatim/tools/refresh.py
View File

@@ -146,6 +146,25 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
return 0
def import_osm_views_geotiff(dsn, data_path, ignore_errors=False):
""" Replaces the OSM views table with new data.
Returns 0 if all was well and 1 if the GeoTIFF file could not
be found. Throws an exception if there was an error reading the file.
"""
datafile = data_path / 'osmviews.tiff'
if not datafile.exists():
return 1
pre_code = """BEGIN;
DROP TABLE IF EXISTS "osmviews";
"""
post_code = "COMMIT"
execute_file(dsn, datafile, ignore_errors=ignore_errors,
pre_code=pre_code, post_code=post_code)
return 0
def recompute_importance(conn: Connection) -> None:
""" Recompute wikipedia links and importance for all entries in placex.

4
settings/env.defaults
View File

@@ -86,6 +86,10 @@ NOMINATIM_TIGER_DATA_PATH=
# When unset, the data is expected to be located in the project directory.
NOMINATIM_WIKIPEDIA_DATA_PATH=
# Directory where to find OSM views GeoTIFF file.
# When unset, the data is expected to be located in the project directory.
NOMINATIM_OSM_VIEWS_DATA_PATH=
# Configuration file for special phrase import.
# OBSOLETE: use `nominatim special-phrases --config <file>` or simply put
# a custom phrase-settings.json into your project directory.

1
test/python/cli/test_cmd_import.py
View File

@@ -40,6 +40,7 @@ class TestCliImportWithDb:
mock_func_factory(nominatim.data.country_info, 'setup_country_tables'),
mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
mock_func_factory(nominatim.tools.refresh, 'import_osm_views_geotiff'),
mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim.tools.database_import, 'load_data'),
mock_func_factory(nominatim.tools.database_import, 'create_tables'),

11
test/python/cli/test_cmd_refresh.py
View File

@@ -24,6 +24,7 @@ class TestRefresh:
@pytest.mark.parametrize("command,func", [
('address-levels', 'load_address_levels_from_config'),
('wiki-data', 'import_wikipedia_articles'),
('osm-views', 'import_osm_views_geotiff')
('importance', 'recompute_importance'),
('website', 'setup_website'),
])
@@ -71,15 +72,21 @@ class TestRefresh:
assert self.call_nominatim('refresh', '--wiki-data') == 1
def test_refresh_osm_views_geotiff_file_not_found(self, monkeypatch):
monkeypatch.setenv('NOMINATIM_OSM_VIEWS_DATA_PATH', 'gjoiergjeroi345Q')
def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
assert self.call_nominatim('refresh', '--osm-views') == 1
def test_refresh_importance_computed_after_wiki_and_osm_views_import(self, monkeypatch):
calls = []
monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
lambda *args, **kwargs: calls.append('import') or 0)
monkeypatch.setattr(nominatim.tools.refresh, 'import_osm_views_geotiff',
lambda *args, **kwargs: calls.append('import') or 0)
monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
lambda *args, **kwargs: calls.append('update'))
assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
assert self.call_nominatim('refresh', '--importance', '--wiki-data', '--osm-views') == 0
assert calls == ['import', 'update']

1
test/python/tools/test_freeze.py
View File

@@ -21,6 +21,7 @@ NOMINATIM_DROP_TABLES = [
'address_levels',
'location_area', 'location_area_country', 'location_area_large_100',
'location_road_1',
'osmviews'
'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways',
'search_name_111',
'wikipedia_article', 'wikipedia_redirect'

15
test/python/tools/test_refresh.py
View File

@@ -17,6 +17,10 @@ def test_refresh_import_wikipedia_not_existing(dsn):
assert refresh.import_wikipedia_articles(dsn, Path('.')) == 1
def test_refresh_import_osm_views_geotiff_not_existing(dsn):
assert refresh.import_osm_views_geotiff(dsn, Path('.')) == 1
@pytest.mark.parametrize("replace", (True, False))
def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, replace):
if replace:
@@ -30,6 +34,17 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r
assert temp_db_cursor.table_rows('wikipedia_redirect') > 0
@pytest.mark.parametrize("replace", (True, False))
def test_refresh_import_osm_views_geotiff(dsn, src_dir, table_factory, temp_db_cursor, replace):
if replace:
table_factory('osmviews')
# use the small osm views GeoTIFF file for the API testdb
assert refresh.import_osm_views_geotiff(dsn, src_dir / 'test' / 'testdb') == 0
assert temp_db_cursor.table_rows('osmviews') > 0
def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor):
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
country_code varchar(2),