safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

Merge pull request #3086 from lonvia/close-connection-on-replication

Browse Source 
Close database connections while waiting for the next update cycle
This commit is contained in:
Sarah Hoffmann
2023-06-19 15:48:00 +02:00
committed by GitHub
parent a7bd39b62a 10e56e0de7
commit 1f83efa8f2
4 changed files with 77 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
docs/admin/Update.md
View File

@@ -59,47 +59,6 @@ imported multiple country extracts and want to keep them
up-to-date, [Advanced installations section](Advanced-Installations.md)
contains instructions to set up and update multiple country extracts.
#### Continuous updates
This is the easiest mode. Simply run the replication command without any
parameters:
nominatim replication
The update application keeps running forever and retrieves and applies
new updates from the server as they are published.
You can run this command as a simple systemd service. Create a service
description like that in `/etc/systemd/system/nominatim-updates.service`:
```
[Unit]
Description=Continuous updates of Nominatim
[Service]
WorkingDirectory=/srv/nominatim
ExecStart=nominatim replication
StandardOutput=append:/var/log/nominatim-updates.log
StandardError=append:/var/log/nominatim-updates.error.log
User=nominatim
Group=nominatim
Type=simple
[Install]
WantedBy=multi-user.target
```
Replace the `WorkingDirectory` with your project directory. Also adapt user
and group names as required.
Now activate the service and start the updates:
```
sudo systemctl daemon-reload
sudo systemctl enable nominatim-updates
sudo systemctl start nominatim-updates
```
#### One-time mode
When the `--once` parameter is given, then Nominatim will download exactly one
@@ -221,3 +180,53 @@ replication catch-up at whatever interval you desire.
updated source with daily updates), use the
continuous update mode. It ensures to re-request the newest update until it
is published.
#### Continuous updates
!!! danger
This mode is no longer recommended to use and will removed in future
releases. systemd is much better
suited for running regular updates. Please refer to the setup
instructions for running one-time mode with systemd above.
This is the easiest mode. Simply run the replication command without any
parameters:
nominatim replication
The update application keeps running forever and retrieves and applies
new updates from the server as they are published.
You can run this command as a simple systemd service. Create a service
description like that in `/etc/systemd/system/nominatim-updates.service`:
```
[Unit]
Description=Continuous updates of Nominatim
[Service]
WorkingDirectory=/srv/nominatim
ExecStart=nominatim replication
StandardOutput=append:/var/log/nominatim-updates.log
StandardError=append:/var/log/nominatim-updates.error.log
User=nominatim
Group=nominatim
Type=simple
[Install]
WantedBy=multi-user.target
```
Replace the `WorkingDirectory` with your project directory. Also adapt user
and group names as required.
Now activate the service and start the updates:
```
sudo systemctl daemon-reload
sudo systemctl enable nominatim-updates
sudo systemctl start nominatim-updates
```

11
nominatim/clicmd/replication.py
View File

@@ -147,10 +147,13 @@ class UpdateReplication:
tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)
dsn = args.config.get_libpq_dsn()
while True:
with connect(args.config.get_libpq_dsn()) as conn:
start = dt.datetime.now(dt.timezone.utc)
state = replication.update(conn, params, socket_timeout=args.socket_timeout)
start = dt.datetime.now(dt.timezone.utc)
state = replication.update(dsn, params, socket_timeout=args.socket_timeout)
with connect(dsn) as conn:
if state is not replication.UpdateState.NO_CHANGES:
status.log_status(conn, start, 'import')
batchdate, _, _ = status.get_status(conn)
@@ -160,7 +163,7 @@ class UpdateReplication:
index_start = dt.datetime.now(dt.timezone.utc)
indexer.index_full(analyse=False)
with connect(args.config.get_libpq_dsn()) as conn:
with connect(dsn) as conn:
status.set_indexed(conn, True)
status.log_status(conn, index_start, 'index')
conn.commit()

20
nominatim/tools/replication.py
View File

@@ -18,7 +18,7 @@ import urllib.request as urlrequest
import requests
from nominatim.db import status
from nominatim.db.connection import Connection
from nominatim.db.connection import Connection, connect
from nominatim.tools.exec_utils import run_osm2pgsql
from nominatim.errors import UsageError
@@ -92,12 +92,14 @@ class UpdateState(Enum):
NO_CHANGES = 3
def update(conn: Connection, options: MutableMapping[str, Any],
def update(dsn: str, options: MutableMapping[str, Any],
socket_timeout: int = 60) -> UpdateState:
""" Update database from the next batch of data. Returns the state of
updates according to `UpdateState`.
"""
startdate, startseq, indexed = status.get_status(conn)
with connect(dsn) as conn:
startdate, startseq, indexed = status.get_status(conn)
conn.commit()
if startseq is None:
LOG.error("Replication not set up. "
@@ -130,12 +132,14 @@ def update(conn: Connection, options: MutableMapping[str, Any],
if endseq is None:
return UpdateState.NO_CHANGES
run_osm2pgsql_updates(conn, options)
with connect(dsn) as conn:
run_osm2pgsql_updates(conn, options)
# Write the current status to the file
endstate = repl.get_state_info(endseq)
status.set_status(conn, endstate.timestamp if endstate else None,
seq=endseq, indexed=False)
# Write the current status to the file
endstate = repl.get_state_info(endseq)
status.set_status(conn, endstate.timestamp if endstate else None,
seq=endseq, indexed=False)
conn.commit()
return UpdateState.UP_TO_DATE

16
test/python/tools/test_replication.py
View File

@@ -100,19 +100,19 @@ def update_options(tmpdir):
import_file=tmpdir / 'foo.osm',
max_diff_size=1)
def test_update_empty_status_table(temp_db_conn):
def test_update_empty_status_table(dsn):
with pytest.raises(UsageError):
nominatim.tools.replication.update(temp_db_conn, {})
nominatim.tools.replication.update(dsn, {})
def test_update_already_indexed(temp_db_conn):
def test_update_already_indexed(temp_db_conn, dsn):
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
assert nominatim.tools.replication.update(dsn, dict(indexed_only=True)) \
== nominatim.tools.replication.UpdateState.MORE_PENDING
def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, dsn, update_options):
date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
status.set_status(temp_db_conn, date, seq=34)
@@ -123,13 +123,13 @@ def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
sleeptime = []
monkeypatch.setattr(time, 'sleep', sleeptime.append)
assert nominatim.tools.replication.update(temp_db_conn, update_options) \
assert nominatim.tools.replication.update(dsn, update_options) \
== nominatim.tools.replication.UpdateState.NO_CHANGES
assert not sleeptime
def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
def test_update_no_data_sleep(monkeypatch, temp_db_conn, dsn, update_options):
date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
status.set_status(temp_db_conn, date, seq=34)
@@ -140,7 +140,7 @@ def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
sleeptime = []
monkeypatch.setattr(time, 'sleep', sleeptime.append)
assert nominatim.tools.replication.update(temp_db_conn, update_options) \
assert nominatim.tools.replication.update(dsn, update_options) \
== nominatim.tools.replication.UpdateState.NO_CHANGES
assert len(sleeptime) == 1