port replication initialisation to Python

nominatim/tools/exec_utils.py

@@ -3,8 +3,13 @@ Helper functions for executing external programs.
 """
 import logging
 import subprocess
+import urllib.request as urlrequest
 from urllib.parse import urlencode
 
+from ..version import NOMINATIM_VERSION
+
+LOG = logging.getLogger()
+
 def run_legacy_script(script, *args, nominatim_env=None, throw_on_fail=False):
     """ Run a Nominatim PHP script with the given arguments.
 
@@ -80,3 +85,16 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
     print(result[content_start + 4:].replace('\\n', '\n'))
 
     return 0
+
+
+def get_url(url):
+    """ Get the contents from the given URL and return it as a UTF-8 string.
+    """
+    headers = {"User-Agent" : "Nominatim/" + NOMINATIM_VERSION}
+
+    try:
+        with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response:
+            return response.read().decode('utf-8')
+    except:
+        LOG.fatal('Failed to load URL: %s', url)
+        raise

nominatim/tools/replication.py

@@ -0,0 +1,34 @@
+"""
+Functions for updating a database from a replication source.
+"""
+import datetime
+import logging
+
+from osmium.replication.server import ReplicationServer
+
+from ..db import status
+
+LOG = logging.getLogger()
+
+def init_replication(conn, base_url):
+    """ Set up replication for the server at the given base URL.
+    """
+    LOG.info("Using replication source: %s", base_url)
+    date = status.compute_database_date(conn)
+
+    # margin of error to make sure we get all data
+    date -= datetime.timedelta(hours=3)
+
+    repl = ReplicationServer(base_url)
+
+    seq = repl.timestamp_to_sequence(date)
+
+    if seq is None:
+        LOG.fatal("Cannot reach the configured replication service '%s'.\n"
+                  "Does the URL point to a directory containing OSM update data?",
+                  base_url)
+        raise RuntimeError("Failed to reach replication service")
+
+    status.set_status(conn, date=date, seq=seq)
+
+    LOG.warning("Updates intialised at sequence %s (%s)", seq, date)