add new replication mode catch-up

This mode gets updates until the server reports no new diffs
anymore.

Also adds additional indexing, when the main indexing step left
a couple of objects to process. This happens only when the
next update is expected to be more than 40min away.
This commit is contained in:
Sarah Hoffmann
2021-10-20 22:05:15 +02:00
parent 12643c5986
commit c1fa70639b
3 changed files with 50 additions and 21 deletions

View File

@@ -91,6 +91,17 @@ class Indexer:
self.num_threads = num_threads
def has_pending(self):
""" Check if any data still needs indexing.
This function must only be used after the import has finished.
Otherwise it will be very expensive.
"""
with connect(self.dsn) as conn:
with conn.cursor() as cur:
cur.execute("SELECT 'a' FROM placex WHERE indexed_status > 0 LIMIT 1")
return cur.rowcount > 0
def index_full(self, analyse=True):
""" Index the complete database. This will first index boundaries
followed by all other objects. When `analyse` is True, then the