call osm2pgsql postprocessing flush_deleted_places() when adding data

nominatim/clicmd/add_data.py

@@ -76,21 +76,25 @@ class UpdateAddData:
 
         osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
         if args.file or args.diff:
-            return add_osm_data.add_data_from_file(cast(str, args.file or args.diff),
+            return add_osm_data.add_data_from_file(args.config.get_libpq_dsn(),
+                                                   cast(str, args.file or args.diff),
                                                    osm2pgsql_params)
 
         if args.node:
-            return add_osm_data.add_osm_object('node', args.node,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'node', args.node,
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.way:
-            return add_osm_data.add_osm_object('way', args.way,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'way', args.way,
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.relation:
-            return add_osm_data.add_osm_object('relation', args.relation,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'relation', args.relation,
                                                args.use_main_api,
                                                osm2pgsql_params)
 

nominatim/tools/add_osm_data.py

@@ -12,23 +12,34 @@ from pathlib import Path
 import logging
 import urllib
 
+from nominatim.db.connection import connect
 from nominatim.tools.exec_utils import run_osm2pgsql, get_url
 
 LOG = logging.getLogger()
 
-def add_data_from_file(fname: str, options: MutableMapping[str, Any]) -> int:
+def _run_osm2pgsql(dsn: str, options: MutableMapping[str, Any]) -> None:
+    run_osm2pgsql(options)
+
+    # Handle deletions
+    with connect(dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute('SELECT flush_deleted_places()')
+        conn.commit()
+
+
+def add_data_from_file(dsn: str, fname: str, options: MutableMapping[str, Any]) -> int:
     """ Adds data from a OSM file to the database. The file may be a normal
         OSM file or a diff file in all formats supported by libosmium.
     """
     options['import_file'] = Path(fname)
     options['append'] = True
-    run_osm2pgsql(options)
+    _run_osm2pgsql(dsn, options)
 
     # No status update. We don't know where the file came from.
     return 0
 
 
-def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
+def add_osm_object(dsn: str, osm_type: str, osm_id: int, use_main_api: bool,
                    options: MutableMapping[str, Any]) -> int:
     """ Add or update a single OSM object from the latest version of the
         API.
@@ -51,6 +62,6 @@ def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
     options['append'] = True
     options['import_data'] = get_url(base_url).encode('utf-8')
 
-    run_osm2pgsql(options)
+    _run_osm2pgsql(dsn, options)
 
     return 0

test/python/tools/test_add_osm_data.py

@@ -24,10 +24,14 @@ class CaptureGetUrl:
         return '<xml></xml>'
 
 
-def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
+@pytest.fixture(autouse=True)
-    table_factory('place', content=((1, ), ))
+def setup_delete_postprocessing(temp_db_cursor):
+    temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION flush_deleted_places()
+                              RETURNS INTEGER AS $$ SELECT 1 $$ LANGUAGE SQL""")
 
-    assert add_osm_data.add_data_from_file(Path('change.osm'), osm2pgsql_options) == 0
+def test_import_osm_file_simple(dsn, table_factory, osm2pgsql_options, capfd):
+
+    assert add_osm_data.add_data_from_file(dsn, Path('change.osm'), osm2pgsql_options) == 0
     captured = capfd.readouterr()
 
     assert '--append' in captured.out
@@ -41,11 +45,11 @@ def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
 @pytest.mark.parametrize("osm_type", ['node', 'way', 'relation'])
 @pytest.mark.parametrize("main_api,url", [(True, 'https://www.openstreetmap.org/api'),
                                           (False, 'https://overpass-api.de/api/interpreter?')])
-def test_import_osm_object_main_api(osm2pgsql_options, monkeypatch, capfd,
+def test_import_osm_object_main_api(dsn, osm2pgsql_options, monkeypatch,
-                                    osm_type, main_api, url):
+                                    capfd, osm_type, main_api, url):
     get_url_mock = CaptureGetUrl(monkeypatch)
 
-    add_osm_data.add_osm_object(osm_type, 4536, main_api, osm2pgsql_options)
+    add_osm_data.add_osm_object(dsn, osm_type, 4536, main_api, osm2pgsql_options)
     captured = capfd.readouterr()
 
     assert get_url_mock.url.startswith(url)