hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-15 19:07:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3833 from lonvia/rework-logging

Browse Source 
Introduce generic query statistics and make log output configurable
This commit is contained in:
Sarah Hoffmann
2025-09-11 08:46:44 +02:00
committed by GitHub
parent 7ac3591433 45a44f1411
commit bf49f6a46f
13 changed files with 248 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/nominatim_api/__init__.py
View File

@@ -24,7 +24,8 @@ from .types import (PlaceID as PlaceID,
Point as Point,
Bbox as Bbox,
GeometryFormat as GeometryFormat,
DataLayer as DataLayer)
DataLayer as DataLayer,
QueryStatistics as QueryStatistics)
from .results import (SourceTable as SourceTable,
AddressLine as AddressLine,
AddressLines as AddressLines,

111
src/nominatim_api/core.py
View File

@@ -217,11 +217,13 @@ class NominatimAPIAsync:
"""
timeout = Timeout(self.request_timeout)
details = ntyp.LookupDetails.from_kwargs(params)
async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
if details.keywords:
await nsearch.make_query_analyzer(conn)
return await get_detailed_place(conn, place, details)
with details.query_stats as qs:
async with self.begin(abs_timeout=timeout.abs) as conn:
qs.log_time('start_query')
conn.set_query_timeout(self.query_timeout)
if details.keywords:
await nsearch.make_query_analyzer(conn)
return await get_detailed_place(conn, place, details)
async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
""" Get simple information about a list of places.
@@ -230,11 +232,13 @@ class NominatimAPIAsync:
"""
timeout = Timeout(self.request_timeout)
details = ntyp.LookupDetails.from_kwargs(params)
async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
if details.keywords:
await nsearch.make_query_analyzer(conn)
return await get_places(conn, places, details)
with details.query_stats as qs:
async with self.begin(abs_timeout=timeout.abs) as conn:
qs.log_time('start_query')
conn.set_query_timeout(self.query_timeout)
if details.keywords:
await nsearch.make_query_analyzer(conn)
return await get_places(conn, places, details)
async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
""" Find a place by its coordinates. Also known as reverse geocoding.
@@ -249,28 +253,32 @@ class NominatimAPIAsync:
timeout = Timeout(self.request_timeout)
details = ntyp.ReverseDetails.from_kwargs(params)
async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
if details.keywords:
await nsearch.make_query_analyzer(conn)
geocoder = ReverseGeocoder(conn, details,
self.reverse_restrict_to_country_area)
return await geocoder.lookup(coord)
with details.query_stats as qs:
async with self.begin(abs_timeout=timeout.abs) as conn:
qs.log_time('start_query')
conn.set_query_timeout(self.query_timeout)
if details.keywords:
await nsearch.make_query_analyzer(conn)
geocoder = ReverseGeocoder(conn, details,
self.reverse_restrict_to_country_area)
return await geocoder.lookup(coord)
async def search(self, query: str, **params: Any) -> SearchResults:
""" Find a place by free-text search. Also known as forward geocoding.
"""
query = query.strip()
if not query:
raise UsageError('Nothing to search for.')
timeout = Timeout(self.request_timeout)
async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
geocoder = nsearch.ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
timeout)
phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p.strip()) for p in query.split(',')]
return await geocoder.lookup(phrases)
details = ntyp.SearchDetails.from_kwargs(params)
with details.query_stats as qs:
query = query.strip()
if not query:
raise UsageError('Nothing to search for.')
async with self.begin(abs_timeout=timeout.abs) as conn:
qs.log_time('start_query')
conn.set_query_timeout(self.query_timeout)
geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p.strip()) for p in query.split(',')]
return await geocoder.lookup(phrases)
async def search_address(self, amenity: Optional[str] = None,
street: Optional[str] = None,
@@ -283,10 +291,8 @@ class NominatimAPIAsync:
""" Find an address using structured search.
"""
timeout = Timeout(self.request_timeout)
async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
details = ntyp.SearchDetails.from_kwargs(params)
details = ntyp.SearchDetails.from_kwargs(params)
with details.query_stats as qs:
phrases: List[nsearch.Phrase] = []
if amenity:
@@ -325,6 +331,9 @@ class NominatimAPIAsync:
if amenity:
details.layers |= ntyp.DataLayer.POI
async with self.begin(abs_timeout=timeout.abs) as conn:
qs.log_time('start_query')
conn.set_query_timeout(self.query_timeout)
geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
return await geocoder.lookup(phrases)
@@ -335,22 +344,24 @@ class NominatimAPIAsync:
The near place may either be given as an unstructured search
query in itself or as coordinates.
"""
if not categories:
return SearchResults()
timeout = Timeout(self.request_timeout)
details = ntyp.SearchDetails.from_kwargs(params)
async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
if near_query:
phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p) for p in near_query.split(',')]
else:
phrases = []
if details.keywords:
await nsearch.make_query_analyzer(conn)
with details.query_stats as qs:
if not categories:
return SearchResults()
geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
return await geocoder.lookup_pois(categories, phrases)
async with self.begin(abs_timeout=timeout.abs) as conn:
qs.log_time('start_query')
conn.set_query_timeout(self.query_timeout)
if near_query:
phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p) for p in near_query.split(',')]
else:
phrases = []
if details.keywords:
await nsearch.make_query_analyzer(conn)
geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
return await geocoder.lookup_pois(categories, phrases)
class NominatimAPI:
@@ -447,6 +458,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
query_stats (QueryStatistics): When given collects statistics
about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -529,6 +542,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
query_stats (QueryStatistics): When given collects statistics
about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -609,6 +624,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
query_stats (QueryStatistics): When given collects statistics
about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -708,6 +725,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
query_stats (QueryStatistics): When given collects statistics
about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -824,6 +843,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
query_stats (QueryStatistics): When given collects statistics
about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -931,6 +952,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
query_stats (QueryStatistics): When given collects statistics
about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.

10
src/nominatim_api/search/geocoder.py
View File

@@ -77,7 +77,9 @@ class ForwardGeocoder:
"""
log().section('Execute database searches')
results: Dict[Any, SearchResult] = {}
qs = self.params.query_stats
qs['search_min_penalty'] = round(searches[0].penalty, 2)
min_ranking = searches[0].penalty + 2.0
prev_penalty = 0.0
for i, search in enumerate(searches):
@@ -93,6 +95,13 @@ class ForwardGeocoder:
if prevresult:
prevresult.accuracy = min(prevresult.accuracy, result.accuracy)
else:
if not results:
qs['search_first_result_round'] = i
spenalty = round(search.penalty, 2)
if 'search_min_result_penalty' not in qs or \
spenalty < qs['search_min_result_penalty']:
qs['search_min_result_penalty'] = spenalty
qs['search_best_penalty_round'] = i
results[rhash] = result
min_ranking = min(min_ranking, result.accuracy * 1.2, 2.0)
log().result_dump('Results', ((r.accuracy, r) for r in lookup_results))
@@ -100,6 +109,7 @@ class ForwardGeocoder:
if self.timeout.is_elapsed():
break
qs['search_rounds'] = i
return SearchResults(results.values())
def pre_filter_results(self, results: SearchResults) -> SearchResults:

9
src/nominatim_api/server/asgi_adaptor.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Base abstraction for implementing based on different ASGI frameworks.
@@ -13,6 +13,7 @@ import math
from ..config import Configuration
from ..core import NominatimAPIAsync
from ..types import QueryStatistics
from ..result_formatting import FormatDispatcher
from .content_types import CONTENT_TEXT
@@ -68,6 +69,12 @@ class ASGIAdaptor(abc.ABC):
""" Return the formatting object to use.
"""
@abc.abstractmethod
def query_stats(self) -> Optional[QueryStatistics]:
""" Return the object for saving query statistics or None if
no statistics are required.
"""
def get_int(self, name: str, default: Optional[int] = None) -> int:
""" Return an input parameter as an int. Raises an exception if
the parameter is given but not in an integer format.

38
src/nominatim_api/server/falcon/server.py
View File

@@ -2,20 +2,21 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the falcon webserver framework.
"""
from typing import Optional, Mapping, Any, List
from typing import Optional, Mapping, Any, List, cast
from pathlib import Path
import datetime as dt
import asyncio
import datetime as dt
from falcon.asgi import App, Request, Response
from ...config import Configuration
from ...core import NominatimAPIAsync
from ...types import QueryStatistics
from ... import v1 as api_impl
from ...result_formatting import FormatDispatcher, load_format_dispatcher
from ... import logging as loglib
@@ -95,6 +96,9 @@ class ParamWrapper(ASGIAdaptor):
def formatting(self) -> FormatDispatcher:
return self._formatter
def query_stats(self) -> Optional[QueryStatistics]:
return cast(Optional[QueryStatistics], getattr(self.request.context, 'query_stats', None))
class EndpointWrapper:
""" Converter for server glue endpoint functions to Falcon request handlers.
@@ -118,13 +122,14 @@ class FileLoggingMiddleware:
""" Middleware to log selected requests into a file.
"""
def __init__(self, file_name: str):
def __init__(self, file_name: str, logstr: str):
self.logstr = logstr + '\n'
self.fd = open(file_name, 'a', buffering=1, encoding='utf8')
async def process_request(self, req: Request, _: Response) -> None:
""" Callback before the request starts timing.
"""
req.context.start = dt.datetime.now(tz=dt.timezone.utc)
req.context.query_stats = QueryStatistics()
async def process_response(self, req: Request, resp: Response,
resource: Optional[EndpointWrapper],
@@ -132,19 +137,22 @@ class FileLoggingMiddleware:
""" Callback after requests writes to the logfile. It only
writes logs for successful requests for search, reverse and lookup.
"""
if not req_succeeded or resource is None or resp.status != 200\
qs = req.context.query_stats
if not req_succeeded or 'start' not in qs\
or resource is None or resp.status != 200\
or resource.name not in ('reverse', 'search', 'lookup', 'details'):
return
finish = dt.datetime.now(tz=dt.timezone.utc)
duration = (finish - req.context.start).total_seconds()
params = req.scope['query_string'].decode('utf8')
start = req.context.start.replace(tzinfo=None)\
.isoformat(sep=' ', timespec='milliseconds')
qs['endpoint'] = resource.name
qs['query_string'] = req.scope['query_string'].decode('utf8')
qs['results_total'] = getattr(resp.context, 'num_results', 0)
for param in ('start', 'end', 'start_query'):
if isinstance(qs.get(param), dt.datetime):
qs[param] = qs[param].replace(tzinfo=None)\
.isoformat(sep=' ', timespec='milliseconds')
self.fd.write(f"[{start}] "
f"{duration:.4f} {getattr(resp.context, 'num_results', 0)} "
f'{resource.name} "{params}"\n')
self.fd.write(self.logstr.format_map(qs))
class APIMiddleware:
@@ -193,7 +201,7 @@ def get_application(project_dir: Path,
middleware: List[Any] = [apimw]
log_file = apimw.config.LOG_FILE
if log_file:
middleware.append(FileLoggingMiddleware(log_file))
middleware.append(FileLoggingMiddleware(log_file, apimw.config.LOG_FORMAT))
app = App(cors_enable=apimw.config.get_bool('CORS_NOACCESSCONTROL'),
middleware=middleware)

35
src/nominatim_api/server/starlette/server.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the starlette webserver framework.
@@ -10,9 +10,9 @@ Server implementation using the starlette webserver framework.
from typing import Any, Optional, Mapping, Callable, cast, Coroutine, Dict, \
Awaitable, AsyncIterator
from pathlib import Path
import datetime as dt
import asyncio
import contextlib
import datetime as dt
from starlette.applications import Starlette
from starlette.routing import Route
@@ -25,6 +25,7 @@ from starlette.middleware.cors import CORSMiddleware
from ...config import Configuration
from ...core import NominatimAPIAsync
from ...types import QueryStatistics
from ... import v1 as api_impl
from ...result_formatting import FormatDispatcher, load_format_dispatcher
from ..asgi_adaptor import ASGIAdaptor, EndpointFunc
@@ -70,6 +71,9 @@ class ParamWrapper(ASGIAdaptor):
def formatting(self) -> FormatDispatcher:
return cast(FormatDispatcher, self.request.app.state.formatter)
def query_stats(self) -> Optional[QueryStatistics]:
return cast(Optional[QueryStatistics], getattr(self.request.state, 'query_stats', None))
def _wrap_endpoint(func: EndpointFunc)\
-> Callable[[Request], Coroutine[Any, Any, Response]]:
@@ -83,33 +87,35 @@ class FileLoggingMiddleware(BaseHTTPMiddleware):
""" Middleware to log selected requests into a file.
"""
def __init__(self, app: Starlette, file_name: str = ''):
def __init__(self, app: Starlette, file_name: str = '', logstr: str = ''):
super().__init__(app)
self.fd = open(file_name, 'a', buffering=1, encoding='utf8')
self.logstr = logstr + '\n'
async def dispatch(self, request: Request,
call_next: RequestResponseEndpoint) -> Response:
start = dt.datetime.now(tz=dt.timezone.utc)
qs = QueryStatistics()
request.state.query_stats = qs
response = await call_next(request)
if response.status_code != 200:
if response.status_code != 200 or 'start' not in qs:
return response
finish = dt.datetime.now(tz=dt.timezone.utc)
for endpoint in ('reverse', 'search', 'lookup', 'details'):
if request.url.path.startswith('/' + endpoint):
qtype = endpoint
qs['endpoint'] = endpoint
break
else:
return response
duration = (finish - start).total_seconds()
params = request.scope['query_string'].decode('utf8')
qs['query_string'] = request.scope['query_string'].decode('utf8')
qs['results_total'] = getattr(request.state, 'num_results', 0)
for param in ('start', 'end', 'start_query'):
if isinstance(qs.get(param), dt.datetime):
qs[param] = qs[param].replace(tzinfo=None)\
.isoformat(sep=' ', timespec='milliseconds')
self.fd.write(f"[{start.replace(tzinfo=None).isoformat(sep=' ', timespec='milliseconds')}] "
f"{duration:.4f} {getattr(request.state, 'num_results', 0)} "
f'{qtype} "{params}"\n')
self.fd.write(self.logstr.format_map(qs))
return response
@@ -143,7 +149,8 @@ def get_application(project_dir: Path,
log_file = config.LOG_FILE
if log_file:
middleware.append(Middleware(FileLoggingMiddleware, file_name=log_file)) # type: ignore
middleware.append(Middleware(FileLoggingMiddleware, file_name=log_file, # type: ignore
logstr=config.LOG_FORMAT))
exceptions: Dict[Any, Callable[[Request, Exception], Awaitable[Response]]] = {
TimeoutError: timeout_error,

56
src/nominatim_api/types.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Complex datatypes used by the Nominatim API.
@@ -11,6 +11,7 @@ from typing import Optional, Union, Tuple, NamedTuple, TypeVar, Type, Dict, \
Any, List, Sequence
from collections import abc
import dataclasses
import datetime as dt
import enum
import math
from struct import unpack
@@ -334,6 +335,54 @@ class DataLayer(enum.Flag):
"""
class QueryStatistics(dict[str, Any]):
""" A specialised dictionary for collecting query statistics.
"""
def __enter__(self) -> 'QueryStatistics':
self.log_time('start')
return self
def __exit__(self, *_: Any) -> None:
self.log_time('end')
self['total_time'] = (self['end'] - self['start']).total_seconds()
if 'start_query' in self:
self['wait_time'] = (self['start_query'] - self['start']).total_seconds()
else:
self['wait_time'] = self['total_time']
self['start_query'] = self['end']
self['query_time'] = self['total_time'] - self['wait_time']
def __missing__(self, key: str) -> str:
return ''
def log_time(self, key: str) -> None:
self[key] = dt.datetime.now(tz=dt.timezone.utc)
class NoQueryStats:
""" Null object to use, when no query statistics are requested.
"""
def __enter__(self) -> 'NoQueryStats':
return self
def __exit__(self, *_: Any) -> None:
pass
def __setitem__(self, key: str, value: Any) -> None:
pass
def __getitem__(self, key: str) -> Any:
return None
def __contains__(self, key: str, default: Any = None) -> bool:
return False
def log_time(self, key: str) -> None:
pass
def format_country(cc: Any) -> List[str]:
""" Extract a list of country codes from the input which may be either
a string or list of strings. Filters out all values that are not
@@ -412,6 +461,11 @@ class LookupDetails:
0.0 means the original geometry is kept. The higher the value, the
more the geometry gets simplified.
"""
query_stats: Union[QueryStatistics, NoQueryStats] = \
dataclasses.field(default_factory=NoQueryStats)
""" Optional QueryStatistics object collecting information about
runtime behaviour of the call.
"""
@classmethod
def from_kwargs(cls: Type[TParam], kwargs: Dict[str, Any]) -> TParam:

6
src/nominatim_api/v1/server_glue.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Generic part of the server implementation of the v1 API.
@@ -165,6 +165,7 @@ async def details_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
geometry_output=(GeometryFormat.GEOJSON
if params.get_bool('polygon_geojson', False)
else GeometryFormat.NONE),
query_stats=params.query_stats()
)
if debug:
@@ -197,6 +198,7 @@ async def reverse_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
details = parse_geometry_details(params, fmt)
details['max_rank'] = helpers.zoom_to_rank(params.get_int('zoom', 18))
details['layers'] = get_layers(params)
details['query_stats'] = params.query_stats()
result = await api.reverse(coord, **details)
@@ -234,6 +236,7 @@ async def lookup_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
fmt = parse_format(params, SearchResults, 'xml')
debug = setup_debugging(params)
details = parse_geometry_details(params, fmt)
details['query_stats'] = params.query_stats()
places = []
for oid in (params.get('osm_ids') or '').split(','):
@@ -302,6 +305,7 @@ async def search_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
debug = setup_debugging(params)
details = parse_geometry_details(params, fmt)
details['query_stats'] = params.query_stats()
details['countries'] = params.get('countrycodes', None)
details['entrances'] = params.get_bool('entrances', False)
details['excluded'] = params.get('exclude_place_ids', None)