mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-03-12 22:04:07 +00:00
split code into submodules
This commit is contained in:
Sarah Hoffmann
0
src/nominatim_db/indexer/__init__.py
Normal file
0
src/nominatim_db/indexer/__init__.py
Normal file
242
src/nominatim_db/indexer/indexer.py
Normal file
242
src/nominatim_db/indexer/indexer.py
Normal file
@@ -0,0 +1,242 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2024 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Main work horse for indexing (computing addresses) the database.
|
||||
"""
|
||||
from typing import Optional, Any, cast
|
||||
import logging
|
||||
import time
|
||||
|
||||
import psycopg2.extras
|
||||
|
||||
from nominatim_core.typing import DictCursorResults
|
||||
from nominatim_core.db.async_connection import DBConnection, WorkerPool
|
||||
from nominatim_core.db.connection import connect, Connection, Cursor
|
||||
from ..tokenizer.base import AbstractTokenizer
|
||||
from .progress import ProgressLogger
|
||||
from . import runners
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
|
||||
class PlaceFetcher:
|
||||
""" Asynchronous connection that fetches place details for processing.
|
||||
"""
|
||||
def __init__(self, dsn: str, setup_conn: Connection) -> None:
|
||||
self.wait_time = 0.0
|
||||
self.current_ids: Optional[DictCursorResults] = None
|
||||
self.conn: Optional[DBConnection] = DBConnection(dsn,
|
||||
cursor_factory=psycopg2.extras.DictCursor)
|
||||
|
||||
with setup_conn.cursor() as cur:
|
||||
# need to fetch those manually because register_hstore cannot
|
||||
# fetch them on an asynchronous connection below.
|
||||
hstore_oid = cur.scalar("SELECT 'hstore'::regtype::oid")
|
||||
hstore_array_oid = cur.scalar("SELECT 'hstore[]'::regtype::oid")
|
||||
|
||||
psycopg2.extras.register_hstore(self.conn.conn, oid=hstore_oid,
|
||||
array_oid=hstore_array_oid)
|
||||
|
||||
def close(self) -> None:
|
||||
""" Close the underlying asynchronous connection.
|
||||
"""
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
self.conn = None
|
||||
|
||||
|
||||
def fetch_next_batch(self, cur: Cursor, runner: runners.Runner) -> bool:
|
||||
""" Send a request for the next batch of places.
|
||||
If details for the places are required, they will be fetched
|
||||
asynchronously.
|
||||
|
||||
Returns true if there is still data available.
|
||||
"""
|
||||
ids = cast(Optional[DictCursorResults], cur.fetchmany(100))
|
||||
|
||||
if not ids:
|
||||
self.current_ids = None
|
||||
return False
|
||||
|
||||
assert self.conn is not None
|
||||
self.current_ids = runner.get_place_details(self.conn, ids)
|
||||
|
||||
return True
|
||||
|
||||
def get_batch(self) -> DictCursorResults:
|
||||
""" Get the next batch of data, previously requested with
|
||||
`fetch_next_batch`.
|
||||
"""
|
||||
assert self.conn is not None
|
||||
assert self.conn.cursor is not None
|
||||
|
||||
if self.current_ids is not None and not self.current_ids:
|
||||
tstart = time.time()
|
||||
self.conn.wait()
|
||||
self.wait_time += time.time() - tstart
|
||||
self.current_ids = cast(Optional[DictCursorResults],
|
||||
self.conn.cursor.fetchall())
|
||||
|
||||
return self.current_ids if self.current_ids is not None else []
|
||||
|
||||
def __enter__(self) -> 'PlaceFetcher':
|
||||
return self
|
||||
|
||||
|
||||
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
|
||||
assert self.conn is not None
|
||||
self.conn.wait()
|
||||
self.close()
|
||||
|
||||
|
||||
class Indexer:
|
||||
""" Main indexing routine.
|
||||
"""
|
||||
|
||||
def __init__(self, dsn: str, tokenizer: AbstractTokenizer, num_threads: int):
|
||||
self.dsn = dsn
|
||||
self.tokenizer = tokenizer
|
||||
self.num_threads = num_threads
|
||||
|
||||
|
||||
def has_pending(self) -> bool:
|
||||
""" Check if any data still needs indexing.
|
||||
This function must only be used after the import has finished.
|
||||
Otherwise it will be very expensive.
|
||||
"""
|
||||
with connect(self.dsn) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 'a' FROM placex WHERE indexed_status > 0 LIMIT 1")
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
def index_full(self, analyse: bool = True) -> None:
|
||||
""" Index the complete database. This will first index boundaries
|
||||
followed by all other objects. When `analyse` is True, then the
|
||||
database will be analysed at the appropriate places to
|
||||
ensure that database statistics are updated.
|
||||
"""
|
||||
with connect(self.dsn) as conn:
|
||||
conn.autocommit = True
|
||||
|
||||
def _analyze() -> None:
|
||||
if analyse:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('ANALYZE')
|
||||
|
||||
if self.index_by_rank(0, 4) > 0:
|
||||
_analyze()
|
||||
|
||||
if self.index_boundaries(0, 30) > 100:
|
||||
_analyze()
|
||||
|
||||
if self.index_by_rank(5, 25) > 100:
|
||||
_analyze()
|
||||
|
||||
if self.index_by_rank(26, 30) > 1000:
|
||||
_analyze()
|
||||
|
||||
if self.index_postcodes() > 100:
|
||||
_analyze()
|
||||
|
||||
|
||||
def index_boundaries(self, minrank: int, maxrank: int) -> int:
|
||||
""" Index only administrative boundaries within the given rank range.
|
||||
"""
|
||||
total = 0
|
||||
LOG.warning("Starting indexing boundaries using %s threads",
|
||||
self.num_threads)
|
||||
|
||||
with self.tokenizer.name_analyzer() as analyzer:
|
||||
for rank in range(max(minrank, 4), min(maxrank, 26)):
|
||||
total += self._index(runners.BoundaryRunner(rank, analyzer))
|
||||
|
||||
return total
|
||||
|
||||
def index_by_rank(self, minrank: int, maxrank: int) -> int:
|
||||
""" Index all entries of placex in the given rank range (inclusive)
|
||||
in order of their address rank.
|
||||
|
||||
When rank 30 is requested then also interpolations and
|
||||
places with address rank 0 will be indexed.
|
||||
"""
|
||||
total = 0
|
||||
maxrank = min(maxrank, 30)
|
||||
LOG.warning("Starting indexing rank (%i to %i) using %i threads",
|
||||
minrank, maxrank, self.num_threads)
|
||||
|
||||
with self.tokenizer.name_analyzer() as analyzer:
|
||||
for rank in range(max(1, minrank), maxrank + 1):
|
||||
total += self._index(runners.RankRunner(rank, analyzer), 20 if rank == 30 else 1)
|
||||
|
||||
if maxrank == 30:
|
||||
total += self._index(runners.RankRunner(0, analyzer))
|
||||
total += self._index(runners.InterpolationRunner(analyzer), 20)
|
||||
|
||||
return total
|
||||
|
||||
|
||||
def index_postcodes(self) -> int:
|
||||
"""Index the entries of the location_postcode table.
|
||||
"""
|
||||
LOG.warning("Starting indexing postcodes using %s threads", self.num_threads)
|
||||
|
||||
return self._index(runners.PostcodeRunner(), 20)
|
||||
|
||||
|
||||
def update_status_table(self) -> None:
|
||||
""" Update the status in the status table to 'indexed'.
|
||||
"""
|
||||
with connect(self.dsn) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('UPDATE import_status SET indexed = true')
|
||||
|
||||
conn.commit()
|
||||
|
||||
def _index(self, runner: runners.Runner, batch: int = 1) -> int:
|
||||
""" Index a single rank or table. `runner` describes the SQL to use
|
||||
for indexing. `batch` describes the number of objects that
|
||||
should be processed with a single SQL statement
|
||||
"""
|
||||
LOG.warning("Starting %s (using batch size %s)", runner.name(), batch)
|
||||
|
||||
with connect(self.dsn) as conn:
|
||||
psycopg2.extras.register_hstore(conn)
|
||||
with conn.cursor() as cur:
|
||||
total_tuples = cur.scalar(runner.sql_count_objects())
|
||||
LOG.debug("Total number of rows: %i", total_tuples)
|
||||
|
||||
conn.commit()
|
||||
|
||||
progress = ProgressLogger(runner.name(), total_tuples)
|
||||
|
||||
if total_tuples > 0:
|
||||
with conn.cursor(name='places') as cur:
|
||||
cur.execute(runner.sql_get_objects())
|
||||
|
||||
with PlaceFetcher(self.dsn, conn) as fetcher:
|
||||
with WorkerPool(self.dsn, self.num_threads) as pool:
|
||||
has_more = fetcher.fetch_next_batch(cur, runner)
|
||||
while has_more:
|
||||
places = fetcher.get_batch()
|
||||
|
||||
# asynchronously get the next batch
|
||||
has_more = fetcher.fetch_next_batch(cur, runner)
|
||||
|
||||
# And insert the current batch
|
||||
for idx in range(0, len(places), batch):
|
||||
part = places[idx:idx + batch]
|
||||
LOG.debug("Processing places: %s", str(part))
|
||||
runner.index_places(pool.next_free_worker(), part)
|
||||
progress.add(len(part))
|
||||
|
||||
LOG.info("Wait time: fetcher: %.2fs, pool: %.2fs",
|
||||
fetcher.wait_time, pool.wait_time)
|
||||
|
||||
conn.commit()
|
||||
|
||||
return progress.done()
|
||||
74
src/nominatim_db/indexer/progress.py
Normal file
74
src/nominatim_db/indexer/progress.py
Normal file
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2024 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Helpers for progress logging.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
INITIAL_PROGRESS = 10
|
||||
|
||||
class ProgressLogger:
|
||||
""" Tracks and prints progress for the indexing process.
|
||||
`name` is the name of the indexing step being tracked.
|
||||
`total` sets up the total number of items that need processing.
|
||||
`log_interval` denotes the interval in seconds at which progress
|
||||
should be reported.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, total: int, log_interval: int = 1) -> None:
|
||||
self.name = name
|
||||
self.total_places = total
|
||||
self.done_places = 0
|
||||
self.rank_start_time = datetime.now()
|
||||
self.log_interval = log_interval
|
||||
self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1
|
||||
|
||||
def add(self, num: int = 1) -> None:
|
||||
""" Mark `num` places as processed. Print a log message if the
|
||||
logging is at least info and the log interval has passed.
|
||||
"""
|
||||
self.done_places += num
|
||||
|
||||
if self.done_places < self.next_info:
|
||||
return
|
||||
|
||||
now = datetime.now()
|
||||
done_time = (now - self.rank_start_time).total_seconds()
|
||||
|
||||
if done_time < 2:
|
||||
self.next_info = self.done_places + INITIAL_PROGRESS
|
||||
return
|
||||
|
||||
places_per_sec = self.done_places / done_time
|
||||
eta = (self.total_places - self.done_places) / places_per_sec
|
||||
|
||||
LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
|
||||
self.done_places, int(done_time),
|
||||
places_per_sec, self.name, eta)
|
||||
|
||||
self.next_info += int(places_per_sec) * self.log_interval
|
||||
|
||||
def done(self) -> int:
|
||||
""" Print final statistics about the progress.
|
||||
"""
|
||||
rank_end_time = datetime.now()
|
||||
|
||||
if rank_end_time == self.rank_start_time:
|
||||
diff_seconds = 0.0
|
||||
places_per_sec = float(self.done_places)
|
||||
else:
|
||||
diff_seconds = (rank_end_time - self.rank_start_time).total_seconds()
|
||||
places_per_sec = self.done_places / diff_seconds
|
||||
|
||||
LOG.warning("Done %d/%d in %d @ %.3f per second - FINISHED %s\n",
|
||||
self.done_places, self.total_places, int(diff_seconds),
|
||||
places_per_sec, self.name)
|
||||
|
||||
return self.done_places
|
||||
196
src/nominatim_db/indexer/runners.py
Normal file
196
src/nominatim_db/indexer/runners.py
Normal file
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2024 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Mix-ins that provide the actual commands for the indexer for various indexing
|
||||
tasks.
|
||||
"""
|
||||
from typing import Any, List
|
||||
import functools
|
||||
|
||||
from psycopg2 import sql as pysql
|
||||
import psycopg2.extras
|
||||
|
||||
from nominatim_core.typing import Query, DictCursorResult, DictCursorResults, Protocol
|
||||
from nominatim_core.db.async_connection import DBConnection
|
||||
from ..data.place_info import PlaceInfo
|
||||
from ..tokenizer.base import AbstractAnalyzer
|
||||
|
||||
# pylint: disable=C0111
|
||||
|
||||
def _mk_valuelist(template: str, num: int) -> pysql.Composed:
|
||||
return pysql.SQL(',').join([pysql.SQL(template)] * num)
|
||||
|
||||
def _analyze_place(place: DictCursorResult, analyzer: AbstractAnalyzer) -> psycopg2.extras.Json:
|
||||
return psycopg2.extras.Json(analyzer.process_place(PlaceInfo(place)))
|
||||
|
||||
|
||||
class Runner(Protocol):
|
||||
def name(self) -> str: ...
|
||||
def sql_count_objects(self) -> Query: ...
|
||||
def sql_get_objects(self) -> Query: ...
|
||||
def get_place_details(self, worker: DBConnection,
|
||||
ids: DictCursorResults) -> DictCursorResults: ...
|
||||
def index_places(self, worker: DBConnection, places: DictCursorResults) -> None: ...
|
||||
|
||||
|
||||
class AbstractPlacexRunner:
|
||||
""" Returns SQL commands for indexing of the placex table.
|
||||
"""
|
||||
SELECT_SQL = pysql.SQL('SELECT place_id FROM placex ')
|
||||
UPDATE_LINE = "(%s, %s::hstore, %s::hstore, %s::int, %s::jsonb)"
|
||||
|
||||
def __init__(self, rank: int, analyzer: AbstractAnalyzer) -> None:
|
||||
self.rank = rank
|
||||
self.analyzer = analyzer
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _index_sql(self, num_places: int) -> pysql.Composed:
|
||||
return pysql.SQL(
|
||||
""" UPDATE placex
|
||||
SET indexed_status = 0, address = v.addr, token_info = v.ti,
|
||||
name = v.name, linked_place_id = v.linked_place_id
|
||||
FROM (VALUES {}) as v(id, name, addr, linked_place_id, ti)
|
||||
WHERE place_id = v.id
|
||||
""").format(_mk_valuelist(AbstractPlacexRunner.UPDATE_LINE, num_places))
|
||||
|
||||
|
||||
def get_place_details(self, worker: DBConnection, ids: DictCursorResults) -> DictCursorResults:
|
||||
worker.perform("""SELECT place_id, extra.*
|
||||
FROM placex, LATERAL placex_indexing_prepare(placex) as extra
|
||||
WHERE place_id IN %s""",
|
||||
(tuple((p[0] for p in ids)), ))
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def index_places(self, worker: DBConnection, places: DictCursorResults) -> None:
|
||||
values: List[Any] = []
|
||||
for place in places:
|
||||
for field in ('place_id', 'name', 'address', 'linked_place_id'):
|
||||
values.append(place[field])
|
||||
values.append(_analyze_place(place, self.analyzer))
|
||||
|
||||
worker.perform(self._index_sql(len(places)), values)
|
||||
|
||||
|
||||
class RankRunner(AbstractPlacexRunner):
|
||||
""" Returns SQL commands for indexing one rank within the placex table.
|
||||
"""
|
||||
|
||||
def name(self) -> str:
|
||||
return f"rank {self.rank}"
|
||||
|
||||
def sql_count_objects(self) -> pysql.Composed:
|
||||
return pysql.SQL("""SELECT count(*) FROM placex
|
||||
WHERE rank_address = {} and indexed_status > 0
|
||||
""").format(pysql.Literal(self.rank))
|
||||
|
||||
def sql_get_objects(self) -> pysql.Composed:
|
||||
return self.SELECT_SQL + pysql.SQL(
|
||||
"""WHERE indexed_status > 0 and rank_address = {}
|
||||
ORDER BY geometry_sector
|
||||
""").format(pysql.Literal(self.rank))
|
||||
|
||||
|
||||
class BoundaryRunner(AbstractPlacexRunner):
|
||||
""" Returns SQL commands for indexing the administrative boundaries
|
||||
of a certain rank.
|
||||
"""
|
||||
|
||||
def name(self) -> str:
|
||||
return f"boundaries rank {self.rank}"
|
||||
|
||||
def sql_count_objects(self) -> pysql.Composed:
|
||||
return pysql.SQL("""SELECT count(*) FROM placex
|
||||
WHERE indexed_status > 0
|
||||
AND rank_search = {}
|
||||
AND class = 'boundary' and type = 'administrative'
|
||||
""").format(pysql.Literal(self.rank))
|
||||
|
||||
def sql_get_objects(self) -> pysql.Composed:
|
||||
return self.SELECT_SQL + pysql.SQL(
|
||||
"""WHERE indexed_status > 0 and rank_search = {}
|
||||
and class = 'boundary' and type = 'administrative'
|
||||
ORDER BY partition, admin_level
|
||||
""").format(pysql.Literal(self.rank))
|
||||
|
||||
|
||||
class InterpolationRunner:
|
||||
""" Returns SQL commands for indexing the address interpolation table
|
||||
location_property_osmline.
|
||||
"""
|
||||
|
||||
def __init__(self, analyzer: AbstractAnalyzer) -> None:
|
||||
self.analyzer = analyzer
|
||||
|
||||
|
||||
def name(self) -> str:
|
||||
return "interpolation lines (location_property_osmline)"
|
||||
|
||||
def sql_count_objects(self) -> str:
|
||||
return """SELECT count(*) FROM location_property_osmline
|
||||
WHERE indexed_status > 0"""
|
||||
|
||||
def sql_get_objects(self) -> str:
|
||||
return """SELECT place_id
|
||||
FROM location_property_osmline
|
||||
WHERE indexed_status > 0
|
||||
ORDER BY geometry_sector"""
|
||||
|
||||
|
||||
def get_place_details(self, worker: DBConnection, ids: DictCursorResults) -> DictCursorResults:
|
||||
worker.perform("""SELECT place_id, get_interpolation_address(address, osm_id) as address
|
||||
FROM location_property_osmline WHERE place_id IN %s""",
|
||||
(tuple((p[0] for p in ids)), ))
|
||||
return []
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _index_sql(self, num_places: int) -> pysql.Composed:
|
||||
return pysql.SQL("""UPDATE location_property_osmline
|
||||
SET indexed_status = 0, address = v.addr, token_info = v.ti
|
||||
FROM (VALUES {}) as v(id, addr, ti)
|
||||
WHERE place_id = v.id
|
||||
""").format(_mk_valuelist("(%s, %s::hstore, %s::jsonb)", num_places))
|
||||
|
||||
|
||||
def index_places(self, worker: DBConnection, places: DictCursorResults) -> None:
|
||||
values: List[Any] = []
|
||||
for place in places:
|
||||
values.extend((place[x] for x in ('place_id', 'address')))
|
||||
values.append(_analyze_place(place, self.analyzer))
|
||||
|
||||
worker.perform(self._index_sql(len(places)), values)
|
||||
|
||||
|
||||
|
||||
class PostcodeRunner(Runner):
|
||||
""" Provides the SQL commands for indexing the location_postcode table.
|
||||
"""
|
||||
|
||||
def name(self) -> str:
|
||||
return "postcodes (location_postcode)"
|
||||
|
||||
|
||||
def sql_count_objects(self) -> str:
|
||||
return 'SELECT count(*) FROM location_postcode WHERE indexed_status > 0'
|
||||
|
||||
|
||||
def sql_get_objects(self) -> str:
|
||||
return """SELECT place_id FROM location_postcode
|
||||
WHERE indexed_status > 0
|
||||
ORDER BY country_code, postcode"""
|
||||
|
||||
|
||||
def get_place_details(self, worker: DBConnection, ids: DictCursorResults) -> DictCursorResults:
|
||||
return ids
|
||||
|
||||
def index_places(self, worker: DBConnection, places: DictCursorResults) -> None:
|
||||
worker.perform(pysql.SQL("""UPDATE location_postcode SET indexed_status = 0
|
||||
WHERE place_id IN ({})""")
|
||||
.format(pysql.SQL(',').join((pysql.Literal(i[0]) for i in places))))
|
||||
Reference in New Issue
Block a user