close DB connection when waiting for next update cycle

nominatim/clicmd/replication.py

@@ -147,10 +147,13 @@ class UpdateReplication:
         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
         indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)
 
+        dsn = args.config.get_libpq_dsn()
+
         while True:
-            with connect(args.config.get_libpq_dsn()) as conn:
+            start = dt.datetime.now(dt.timezone.utc)
-                start = dt.datetime.now(dt.timezone.utc)
+            state = replication.update(dsn, params, socket_timeout=args.socket_timeout)
-                state = replication.update(conn, params, socket_timeout=args.socket_timeout)
+
+            with connect(dsn) as conn:
                 if state is not replication.UpdateState.NO_CHANGES:
                     status.log_status(conn, start, 'import')
                 batchdate, _, _ = status.get_status(conn)
@@ -160,7 +163,7 @@ class UpdateReplication:
                 index_start = dt.datetime.now(dt.timezone.utc)
                 indexer.index_full(analyse=False)
 
-                with connect(args.config.get_libpq_dsn()) as conn:
+                with connect(dsn) as conn:
                     status.set_indexed(conn, True)
                     status.log_status(conn, index_start, 'index')
                     conn.commit()

nominatim/tools/replication.py

@@ -18,7 +18,7 @@ import urllib.request as urlrequest
 
 import requests
 from nominatim.db import status
-from nominatim.db.connection import Connection
+from nominatim.db.connection import Connection, connect
 from nominatim.tools.exec_utils import run_osm2pgsql
 from nominatim.errors import UsageError
 
@@ -92,12 +92,14 @@ class UpdateState(Enum):
     NO_CHANGES = 3
 
 
-def update(conn: Connection, options: MutableMapping[str, Any],
+def update(dsn: str, options: MutableMapping[str, Any],
            socket_timeout: int = 60) -> UpdateState:
     """ Update database from the next batch of data. Returns the state of
         updates according to `UpdateState`.
     """
-    startdate, startseq, indexed = status.get_status(conn)
+    with connect(dsn) as conn:
+        startdate, startseq, indexed = status.get_status(conn)
+        conn.commit()
 
     if startseq is None:
         LOG.error("Replication not set up. "
@@ -130,12 +132,14 @@ def update(conn: Connection, options: MutableMapping[str, Any],
         if endseq is None:
             return UpdateState.NO_CHANGES
 
-        run_osm2pgsql_updates(conn, options)
+        with connect(dsn) as conn:
+            run_osm2pgsql_updates(conn, options)
 
-        # Write the current status to the file
+            # Write the current status to the file
-        endstate = repl.get_state_info(endseq)
+            endstate = repl.get_state_info(endseq)
-        status.set_status(conn, endstate.timestamp if endstate else None,
+            status.set_status(conn, endstate.timestamp if endstate else None,
-                          seq=endseq, indexed=False)
+                              seq=endseq, indexed=False)
+            conn.commit()
 
     return UpdateState.UP_TO_DATE
 

test/python/tools/test_replication.py

@@ -100,19 +100,19 @@ def update_options(tmpdir):
                 import_file=tmpdir / 'foo.osm',
                 max_diff_size=1)
 
-def test_update_empty_status_table(temp_db_conn):
+def test_update_empty_status_table(dsn):
     with pytest.raises(UsageError):
-        nominatim.tools.replication.update(temp_db_conn, {})
+        nominatim.tools.replication.update(dsn, {})
 
 
-def test_update_already_indexed(temp_db_conn):
+def test_update_already_indexed(temp_db_conn, dsn):
     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
 
-    assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
+    assert nominatim.tools.replication.update(dsn, dict(indexed_only=True)) \
              == nominatim.tools.replication.UpdateState.MORE_PENDING
 
 
-def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
+def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, dsn, update_options):
     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
     status.set_status(temp_db_conn, date, seq=34)
 
@@ -123,13 +123,13 @@ def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
     sleeptime = []
     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 
-    assert nominatim.tools.replication.update(temp_db_conn, update_options) \
+    assert nominatim.tools.replication.update(dsn, update_options) \
              == nominatim.tools.replication.UpdateState.NO_CHANGES
 
     assert not sleeptime
 
 
-def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
+def test_update_no_data_sleep(monkeypatch, temp_db_conn, dsn, update_options):
     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
     status.set_status(temp_db_conn, date, seq=34)
 
@@ -140,7 +140,7 @@ def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
     sleeptime = []
     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 
-    assert nominatim.tools.replication.update(temp_db_conn, update_options) \
+    assert nominatim.tools.replication.update(dsn, update_options) \
              == nominatim.tools.replication.UpdateState.NO_CHANGES
 
     assert len(sleeptime) == 1