hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-11 05:14:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix style issue found by flake8

Browse Source
This commit is contained in:
Sarah Hoffmann
2024-11-10 22:47:14 +01:00
parent 8c14df55a6
commit 1f07967787
112 changed files with 656 additions and 1109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/nominatim_db/indexer/indexer.py
View File

@@ -21,6 +21,7 @@ from . import runners
LOG = logging.getLogger()
class Indexer:
""" Main indexing routine.
"""
@@ -30,7 +31,6 @@ class Indexer:
self.tokenizer = tokenizer
self.num_threads = num_threads
def has_pending(self) -> bool:
""" Check if any data still needs indexing.
This function must only be used after the import has finished.
@@ -41,7 +41,6 @@ class Indexer:
cur.execute("SELECT 'a' FROM placex WHERE indexed_status > 0 LIMIT 1")
return cur.rowcount > 0
async def index_full(self, analyse: bool = True) -> None:
""" Index the complete database. This will first index boundaries
followed by all other objects. When `analyse` is True, then the
@@ -75,7 +74,6 @@ class Indexer:
if not self.has_pending():
break
async def index_boundaries(self, minrank: int, maxrank: int) -> int:
""" Index only administrative boundaries within the given rank range.
"""
@@ -138,7 +136,6 @@ class Indexer:
(minrank, maxrank))
total_tuples = {row.rank_address: row.count for row in cur}
with self.tokenizer.name_analyzer() as analyzer:
for rank in range(max(1, minrank), maxrank + 1):
if rank >= 30:
@@ -156,7 +153,6 @@ class Indexer:
return total
async def index_postcodes(self) -> int:
"""Index the entries of the location_postcode table.
"""
@@ -164,7 +160,6 @@ class Indexer:
return await self._index(runners.PostcodeRunner(), batch=20)
def update_status_table(self) -> None:
""" Update the status in the status table to 'indexed'.
"""
@@ -193,7 +188,7 @@ class Indexer:
if total_tuples > 0:
async with await psycopg.AsyncConnection.connect(
self.dsn, row_factory=psycopg.rows.dict_row) as aconn,\
self.dsn, row_factory=psycopg.rows.dict_row) as aconn, \
QueryPool(self.dsn, self.num_threads, autocommit=True) as pool:
fetcher_time = 0.0
tstart = time.time()
@@ -224,7 +219,6 @@ class Indexer:
return progress.done()
def _prepare_indexing(self, runner: runners.Runner) -> int:
with connect(self.dsn) as conn:
hstore_info = psycopg.types.TypeInfo.fetch(conn, "hstore")

1
src/nominatim_db/indexer/progress.py
View File

@@ -14,6 +14,7 @@ LOG = logging.getLogger()
INITIAL_PROGRESS = 10
class ProgressLogger:
""" Tracks and prints progress for the indexing process.
`name` is the name of the indexing step being tracked.

14
src/nominatim_db/indexer/runners.py
View File

@@ -19,11 +19,11 @@ from ..typing import Protocol
from ..data.place_info import PlaceInfo
from ..tokenizer.base import AbstractAnalyzer
# pylint: disable=C0111
def _mk_valuelist(template: str, num: int) -> pysql.Composed:
return pysql.SQL(',').join([pysql.SQL(template)] * num)
def _analyze_place(place: DictRow, analyzer: AbstractAnalyzer) -> Json:
return Json(analyzer.process_place(PlaceInfo(place)))
@@ -41,6 +41,7 @@ SELECT_SQL = pysql.SQL("""SELECT place_id, extra.*
LATERAL placex_indexing_prepare(px) as extra """)
UPDATE_LINE = "(%s, %s::hstore, %s::hstore, %s::int, %s::jsonb)"
class AbstractPlacexRunner:
""" Returns SQL commands for indexing of the placex table.
"""
@@ -49,7 +50,6 @@ class AbstractPlacexRunner:
self.rank = rank
self.analyzer = analyzer
def index_places_query(self, batch_size: int) -> Query:
return pysql.SQL(
""" UPDATE placex
@@ -59,7 +59,6 @@ class AbstractPlacexRunner:
WHERE place_id = v.id
""").format(_mk_valuelist(UPDATE_LINE, batch_size))
def index_places_params(self, place: DictRow) -> Sequence[Any]:
return (place['place_id'],
place['name'],
@@ -118,7 +117,6 @@ class InterpolationRunner:
def __init__(self, analyzer: AbstractAnalyzer) -> None:
self.analyzer = analyzer
def name(self) -> str:
return "interpolation lines (location_property_osmline)"
@@ -126,14 +124,12 @@ class InterpolationRunner:
return """SELECT count(*) FROM location_property_osmline
WHERE indexed_status > 0"""
def sql_get_objects(self) -> Query:
return """SELECT place_id, get_interpolation_address(address, osm_id) as address
FROM location_property_osmline
WHERE indexed_status > 0
ORDER BY geometry_sector"""
def index_places_query(self, batch_size: int) -> Query:
return pysql.SQL("""UPDATE location_property_osmline
SET indexed_status = 0, address = v.addr, token_info = v.ti
@@ -141,13 +137,11 @@ class InterpolationRunner:
WHERE place_id = v.id
""").format(_mk_valuelist("(%s, %s::hstore, %s::jsonb)", batch_size))
def index_places_params(self, place: DictRow) -> Sequence[Any]:
return (place['place_id'], place['address'],
_analyze_place(place, self.analyzer))
class PostcodeRunner(Runner):
""" Provides the SQL commands for indexing the location_postcode table.
"""
@@ -155,22 +149,18 @@ class PostcodeRunner(Runner):
def name(self) -> str:
return "postcodes (location_postcode)"
def sql_count_objects(self) -> Query:
return 'SELECT count(*) FROM location_postcode WHERE indexed_status > 0'
def sql_get_objects(self) -> Query:
return """SELECT place_id FROM location_postcode
WHERE indexed_status > 0
ORDER BY country_code, postcode"""
def index_places_query(self, batch_size: int) -> Query:
return pysql.SQL("""UPDATE location_postcode SET indexed_status = 0
WHERE place_id IN ({})""")\
.format(pysql.SQL(',').join((pysql.Placeholder() for _ in range(batch_size))))
def index_places_params(self, place: DictRow) -> Sequence[Any]:
return (place['place_id'], )