hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-14 18:37:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2957 from lonvia/reorganise-api-module

Browse Source 
Assorted improvements to the new api library module
This commit is contained in:
Sarah Hoffmann
2023-01-25 09:29:31 +01:00
committed by GitHub
parent 3cc357bffa f85b0c6208
commit dfcb24061e
22 changed files with 580 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci-tests.yml vendored
View File

@@ -107,7 +107,7 @@ jobs:
if: matrix.flavour == 'oldstuff'
- name: Install Python webservers
run: pip3 install falcon sanic sanic-testing starlette
run: pip3 install falcon sanic sanic-testing sanic-cors starlette
- name: Install latest pylint/mypy
run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs

3
.mypy.ini
View File

@@ -1,6 +1,9 @@
[mypy]
plugins = sqlalchemy.ext.mypy.plugin
[mypy-sanic_cors.*]
ignore_missing_imports = True
[mypy-icu.*]
ignore_missing_imports = True

2
docs/admin/Installation.md
View File

@@ -66,7 +66,7 @@ For running the experimental Python frontend:
* one of the following web frameworks:
* [falcon](https://falconframework.org/) (3.0+)
* [sanic](https://sanic.dev)
* [sanic](https://sanic.dev) and (optionally) [sanic-cors](https://github.com/ashleysommer/sanic-cors)
* [starlette](https://www.starlette.io/)
* [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks)

96
nominatim/api.py
View File

@@ -1,96 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Implementation of classes for API access via libraries.
"""
from typing import Mapping, Optional, cast, Any
import asyncio
from pathlib import Path
from sqlalchemy import text, event
from sqlalchemy.engine.url import URL
from sqlalchemy.ext.asyncio import create_async_engine
import asyncpg
from nominatim.config import Configuration
from nominatim.apicmd.status import get_status, StatusResult
class NominatimAPIAsync:
""" API loader asynchornous version.
"""
def __init__(self, project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> None:
self.config = Configuration(project_dir, environ)
dsn = self.config.get_database_params()
dburl = URL.create(
'postgresql+asyncpg',
database=dsn.get('dbname'),
username=dsn.get('user'), password=dsn.get('password'),
host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None,
query={k: v for k, v in dsn.items()
if k not in ('user', 'password', 'dbname', 'host', 'port')})
self.engine = create_async_engine(
dburl, future=True,
connect_args={'server_settings': {
'DateStyle': 'sql,european',
'max_parallel_workers_per_gather': '0'
}})
asyncio.get_event_loop().run_until_complete(self._query_server_version())
asyncio.get_event_loop().run_until_complete(self.close())
if self.server_version >= 110000:
@event.listens_for(self.engine.sync_engine, "connect") # type: ignore[misc]
def _on_connect(dbapi_con: Any, _: Any) -> None:
cursor = dbapi_con.cursor()
cursor.execute("SET jit_above_cost TO '-1'")
async def _query_server_version(self) -> None:
try:
async with self.engine.begin() as conn:
result = await conn.scalar(text('SHOW server_version_num'))
self.server_version = int(cast(str, result))
except asyncpg.PostgresError:
self.server_version = 0
async def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
called, new connections are created.
"""
await self.engine.dispose()
async def status(self) -> StatusResult:
""" Return the status of the database.
"""
return await get_status(self.engine)
class NominatimAPI:
""" API loader, synchronous version.
"""
def __init__(self, project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> None:
self.async_api = NominatimAPIAsync(project_dir, environ)
def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
called, new connections are created.
"""
asyncio.get_event_loop().run_until_complete(self.async_api.close())
def status(self) -> StatusResult:
""" Return the status of the database.
"""
return asyncio.get_event_loop().run_until_complete(self.async_api.status())

19
nominatim/api/__init__.py Normal file
View File

@@ -0,0 +1,19 @@
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
The public interface of the Nominatim library.
Classes and functions defined in this file are considered stable. Always
import from this file, not from the source files directly.
"""
# See also https://github.com/PyCQA/pylint/issues/6006
# pylint: disable=useless-import-alias
from nominatim.api.core import (NominatimAPI as NominatimAPI,
NominatimAPIAsync as NominatimAPIAsync)
from nominatim.api.status import (StatusResult as StatusResult)

139
nominatim/api/core.py Normal file
View File

@@ -0,0 +1,139 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Implementation of classes for API access via libraries.
"""
from typing import Mapping, Optional, Any, AsyncIterator
import asyncio
import contextlib
from pathlib import Path
import sqlalchemy as sa
import sqlalchemy.ext.asyncio as sa_asyncio
import asyncpg
from nominatim.config import Configuration
from nominatim.api.status import get_status, StatusResult
class NominatimAPIAsync:
""" API loader asynchornous version.
"""
def __init__(self, project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> None:
self.config = Configuration(project_dir, environ)
self.server_version = 0
self._engine_lock = asyncio.Lock()
self._engine: Optional[sa_asyncio.AsyncEngine] = None
async def setup_database(self) -> None:
""" Set up the engine and connection parameters.
This function will be implicitly called when the database is
accessed for the first time. You may also call it explicitly to
avoid that the first call is delayed by the setup.
"""
async with self._engine_lock:
if self._engine:
return
dsn = self.config.get_database_params()
dburl = sa.engine.URL.create(
'postgresql+asyncpg',
database=dsn.get('dbname'),
username=dsn.get('user'), password=dsn.get('password'),
host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None,
query={k: v for k, v in dsn.items()
if k not in ('user', 'password', 'dbname', 'host', 'port')})
engine = sa_asyncio.create_async_engine(
dburl, future=True,
connect_args={'server_settings': {
'DateStyle': 'sql,european',
'max_parallel_workers_per_gather': '0'
}})
try:
async with engine.begin() as conn:
result = await conn.scalar(sa.text('SHOW server_version_num'))
self.server_version = int(result)
except asyncpg.PostgresError:
self.server_version = 0
if self.server_version >= 110000:
@sa.event.listens_for(engine.sync_engine, "connect") # type: ignore[misc]
def _on_connect(dbapi_con: Any, _: Any) -> None:
cursor = dbapi_con.cursor()
cursor.execute("SET jit_above_cost TO '-1'")
# Make sure that all connections get the new settings
await self.close()
self._engine = engine
async def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
called, new connections are created.
"""
if self._engine is not None:
await self._engine.dispose()
@contextlib.asynccontextmanager
async def begin(self) -> AsyncIterator[sa_asyncio.AsyncConnection]:
""" Create a new connection with automatic transaction handling.
This function may be used to get low-level access to the database.
Refer to the documentation of SQLAlchemy for details how to use
the connection object.
"""
if self._engine is None:
await self.setup_database()
assert self._engine is not None
async with self._engine.begin() as conn:
yield conn
async def status(self) -> StatusResult:
""" Return the status of the database.
"""
try:
async with self.begin() as conn:
status = await get_status(conn)
except asyncpg.PostgresError:
return StatusResult(700, 'Database connection failed')
return status
class NominatimAPI:
""" API loader, synchronous version.
"""
def __init__(self, project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> None:
self._loop = asyncio.new_event_loop()
self._async_api = NominatimAPIAsync(project_dir, environ)
def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
called, new connections are created.
"""
self._loop.run_until_complete(self._async_api.close())
self._loop.close()
def status(self) -> StatusResult:
""" Return the status of the database.
"""
return self._loop.run_until_complete(self._async_api.status())

59
nominatim/result_formatter/base.py → nominatim/api/result_formatting.py
View File

@@ -2,49 +2,21 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Helper classes and function for writing result formatting modules.
Helper classes and functions for formating results into API responses.
"""
from typing import Type, TypeVar, Dict, Mapping, List, Callable, Generic, Any
from typing import Type, TypeVar, Dict, List, Callable, Any
from collections import defaultdict
T = TypeVar('T') # pylint: disable=invalid-name
FormatFunc = Callable[[T], str]
class ResultFormatter(Generic[T]):
""" This class dispatches format calls to the appropriate formatting
function previously defined with the `format_func` decorator.
"""
def __init__(self, funcs: Mapping[str, FormatFunc[T]]) -> None:
self.functions = funcs
def list_formats(self) -> List[str]:
""" Return a list of formats supported by this formatter.
"""
return list(self.functions.keys())
def supports_format(self, fmt: str) -> bool:
""" Check if the given format is supported by this formatter.
"""
return fmt in self.functions
def format(self, result: T, fmt: str) -> str:
""" Convert the given result into a string using the given format.
The format is expected to be in the list returned by
`list_formats()`.
"""
return self.functions[fmt](result)
class FormatDispatcher:
""" A factory class for result formatters.
""" Helper class to conveniently create formatting functions in
a module using decorators.
"""
def __init__(self) -> None:
@@ -63,7 +35,22 @@ class FormatDispatcher:
return decorator
def __call__(self, result_class: Type[T]) -> ResultFormatter[T]:
""" Create an instance of a format class for the given result type.
def list_formats(self, result_type: Type[Any]) -> List[str]:
""" Return a list of formats supported by this formatter.
"""
return ResultFormatter(self.format_functions[result_class])
return list(self.format_functions[result_type].keys())
def supports_format(self, result_type: Type[Any], fmt: str) -> bool:
""" Check if the given format is supported by this formatter.
"""
return fmt in self.format_functions[result_type]
def format_result(self, result: Any, fmt: str) -> str:
""" Convert the given result into a string using the given format.
The format is expected to be in the list returned by
`list_formats()`.
"""
return self.format_functions[type(result)][fmt](result)

19
nominatim/apicmd/status.py → nominatim/api/status.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Classes and function releated to status call.
@@ -10,8 +10,8 @@ Classes and function releated to status call.
from typing import Optional, cast
import datetime as dt
import sqlalchemy as sqla
from sqlalchemy.ext.asyncio.engine import AsyncEngine, AsyncConnection
import sqlalchemy as sa
from sqlalchemy.ext.asyncio.engine import AsyncConnection
import asyncpg
from nominatim import version
@@ -31,7 +31,7 @@ class StatusResult:
async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]:
""" Query the database date.
"""
sql = sqla.text('SELECT lastimportdate FROM import_status LIMIT 1')
sql = sa.text('SELECT lastimportdate FROM import_status LIMIT 1')
result = await conn.execute(sql)
for row in result:
@@ -41,8 +41,8 @@ async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]:
async def _get_database_version(conn: AsyncConnection) -> Optional[version.NominatimVersion]:
sql = sqla.text("""SELECT value FROM nominatim_properties
WHERE property = 'database_version'""")
sql = sa.text("""SELECT value FROM nominatim_properties
WHERE property = 'database_version'""")
result = await conn.execute(sql)
for row in result:
@@ -51,14 +51,13 @@ async def _get_database_version(conn: AsyncConnection) -> Optional[version.Nomin
return None
async def get_status(engine: AsyncEngine) -> StatusResult:
async def get_status(conn: AsyncConnection) -> StatusResult:
""" Execute a status API call.
"""
status = StatusResult(0, 'OK')
try:
async with engine.begin() as conn:
status.data_updated = await _get_database_date(conn)
status.database_version = await _get_database_version(conn)
status.data_updated = await _get_database_date(conn)
status.database_version = await _get_database_version(conn)
except asyncpg.PostgresError:
return StatusResult(700, 'Database connection failed')

21
nominatim/api/v1/__init__.py Normal file
View File

@@ -0,0 +1,21 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Implementation of API version v1 (aka the legacy version).
"""
#pylint: disable=useless-import-alias
from nominatim.api.v1.server_glue import (ASGIAdaptor as ASGIAdaptor,
EndpointFunc as EndpointFunc,
ROUTES as ROUTES)
import nominatim.api.v1.format as _format
list_formats = _format.dispatch.list_formats
supports_format = _format.dispatch.supports_format
format_result = _format.dispatch.format_result

12
nominatim/result_formatter/v1.py → nominatim/api/v1/format.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Output formatters for API version v1.
@@ -11,12 +11,12 @@ from typing import Dict, Any
from collections import OrderedDict
import json
from nominatim.result_formatter.base import FormatDispatcher
from nominatim.apicmd.status import StatusResult
from nominatim.api.result_formatting import FormatDispatcher
from nominatim.api import StatusResult
create = FormatDispatcher()
dispatch = FormatDispatcher()
@create.format_func(StatusResult, 'text')
@dispatch.format_func(StatusResult, 'text')
def _format_status_text(result: StatusResult) -> str:
if result.status:
return f"ERROR: {result.message}"
@@ -24,7 +24,7 @@ def _format_status_text(result: StatusResult) -> str:
return 'OK'
@create.format_func(StatusResult, 'json')
@dispatch.format_func(StatusResult, 'json')
def _format_status_json(result: StatusResult) -> str:
out: Dict[str, Any] = OrderedDict()
out['status'] = result.status

153
nominatim/api/v1/server_glue.py Normal file
View File

@@ -0,0 +1,153 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Generic part of the server implementation of the v1 API.
Combine with the scaffolding provided for the various Python ASGI frameworks.
"""
from typing import Optional, Any, Type, Callable
import abc
import nominatim.api as napi
from nominatim.api.v1.format import dispatch as formatting
CONTENT_TYPE = {
'text': 'text/plain; charset=utf-8',
'xml': 'text/xml; charset=utf-8',
'jsonp': 'application/javascript'
}
class ASGIAdaptor(abc.ABC):
""" Adapter class for the different ASGI frameworks.
Wraps functionality over concrete requests and responses.
"""
@abc.abstractmethod
def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
""" Return an input parameter as a string. If the parameter was
not provided, return the 'default' value.
"""
@abc.abstractmethod
def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
""" Return a HTTP header parameter as a string. If the parameter was
not provided, return the 'default' value.
"""
@abc.abstractmethod
def error(self, msg: str) -> Exception:
""" Construct an appropriate exception from the given error message.
The exception must result in a HTTP 400 error.
"""
@abc.abstractmethod
def create_response(self, status: int, output: str, content_type: str) -> Any:
""" Create a response from the given parameters. The result will
be returned by the endpoint functions. The adaptor may also
return None when the response is created internally with some
different means.
The response must return the HTTP given status code 'status', set
the HTTP content-type headers to the string provided and the
body of the response to 'output'.
"""
def build_response(self, output: str, media_type: str, status: int = 200) -> Any:
""" Create a response from the given output. Wraps a JSONP function
around the response, if necessary.
"""
if media_type == 'json' and status == 200:
jsonp = self.get('json_callback')
if jsonp is not None:
if any(not part.isidentifier() for part in jsonp.split('.')):
raise self.error('Invalid json_callback value')
output = f"{jsonp}({output})"
media_type = 'jsonp'
return self.create_response(status, output,
CONTENT_TYPE.get(media_type, 'application/json'))
def get_int(self, name: str, default: Optional[int] = None) -> int:
""" Return an input parameter as an int. Raises an exception if
the parameter is given but not in an integer format.
If 'default' is given, then it will be returned when the parameter
is missing completely. When 'default' is None, an error will be
raised on a missing parameter.
"""
value = self.get(name)
if value is None:
if default is not None:
return default
raise self.error(f"Parameter '{name}' missing.")
try:
return int(value)
except ValueError as exc:
raise self.error(f"Parameter '{name}' must be a number.") from exc
def get_bool(self, name: str, default: Optional[bool] = None) -> bool:
""" Return an input parameter as bool. Only '0' is accepted as
an input for 'false' all other inputs will be interpreted as 'true'.
If 'default' is given, then it will be returned when the parameter
is missing completely. When 'default' is None, an error will be
raised on a missing parameter.
"""
value = self.get(name)
if value is None:
if default is not None:
return default
raise self.error(f"Parameter '{name}' missing.")
return value != '0'
def parse_format(params: ASGIAdaptor, result_type: Type[Any], default: str) -> str:
""" Get and check the 'format' parameter and prepare the formatter.
`fmtter` is a formatter and `default` the
format value to assume when no parameter is present.
"""
fmt = params.get('format', default=default)
assert fmt is not None
if not formatting.supports_format(result_type, fmt):
raise params.error("Parameter 'format' must be one of: " +
', '.join(formatting.list_formats(result_type)))
return fmt
async def status_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any:
""" Server glue for /status endpoint. See API docs for details.
"""
result = await api.status()
fmt = parse_format(params, napi.StatusResult, 'text')
if fmt == 'text' and result.status:
status_code = 500
else:
status_code = 200
return params.build_response(formatting.format_result(result, fmt), fmt,
status=status_code)
EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any]
ROUTES = [
('status', status_endpoint)
]

0
nominatim/apicmd/__init__.py
View File

2
nominatim/cli.py
View File

@@ -236,7 +236,7 @@ class AdminServe:
server_module = importlib.import_module('nominatim.server.sanic.server')
app = server_module.get_application(args.project_dir)
app.run(host=host, port=port, debug=True)
app.run(host=host, port=port, debug=True, single_process=True)
else:
import uvicorn # pylint: disable=import-outside-toplevel

11
nominatim/clicmd/api.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Subcommand definitions for API calls from the command line.
@@ -14,9 +14,8 @@ import logging
from nominatim.tools.exec_utils import run_api_script
from nominatim.errors import UsageError
from nominatim.clicmd.args import NominatimArgs
from nominatim.api import NominatimAPI
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
from nominatim.api import NominatimAPI, StatusResult
import nominatim.api.v1 as api_output
# Do not repeat documentation of subcommand classes.
# pylint: disable=C0111
@@ -277,7 +276,7 @@ class APIStatus:
"""
def add_args(self, parser: argparse.ArgumentParser) -> None:
formats = formatting.create(StatusResult).list_formats()
formats = api_output.list_formats(StatusResult)
group = parser.add_argument_group('API parameters')
group.add_argument('--format', default=formats[0], choices=formats,
help='Format of result')
@@ -285,5 +284,5 @@ class APIStatus:
def run(self, args: NominatimArgs) -> int:
status = NominatimAPI(args.project_dir).status()
print(formatting.create(StatusResult).format(status, args.format))
print(api_output.format_result(status, args.format))
return 0

0
nominatim/result_formatter/__init__.py
View File

103
nominatim/server/falcon/server.py
View File

@@ -2,81 +2,76 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the falcon webserver framework.
"""
from typing import Type, Any, Optional, Mapping
from typing import Optional, Mapping, cast
from pathlib import Path
import falcon
import falcon.asgi
from falcon.asgi import App, Request, Response
from nominatim.api import NominatimAPIAsync
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
import nominatim.api.v1 as api_impl
CONTENT_TYPE = {
'text': falcon.MEDIA_TEXT,
'xml': falcon.MEDIA_XML
}
class NominatimV1:
""" Implementation of V1 version of the Nominatim API.
class ParamWrapper(api_impl.ASGIAdaptor):
""" Adaptor class for server glue to Falcon framework.
"""
def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]]) -> None:
self.api = NominatimAPIAsync(project_dir, environ)
self.formatters = {}
for rtype in (StatusResult, ):
self.formatters[rtype] = formatting.create(rtype)
def __init__(self, req: Request, resp: Response) -> None:
self.request = req
self.response = resp
def parse_format(self, req: falcon.asgi.Request, rtype: Type[Any], default: str) -> None:
""" Get and check the 'format' parameter and prepare the formatter.
`rtype` describes the expected return type and `default` the
format value to assume when no parameter is present.
def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
return cast(Optional[str], self.request.get_param(name, default=default))
def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
return cast(Optional[str], self.request.get_header(name, default=default))
def error(self, msg: str) -> falcon.HTTPBadRequest:
return falcon.HTTPBadRequest(description=msg)
def create_response(self, status: int, output: str, content_type: str) -> None:
self.response.status = status
self.response.text = output
self.response.content_type = content_type
class EndpointWrapper:
""" Converter for server glue endpoint functions to Falcon request handlers.
"""
def __init__(self, func: api_impl.EndpointFunc, api: NominatimAPIAsync) -> None:
self.func = func
self.api = api
async def on_get(self, req: Request, resp: Response) -> None:
""" Implementation of the endpoint.
"""
req.context.format = req.get_param('format', default=default)
req.context.formatter = self.formatters[rtype]
if not req.context.formatter.supports_format(req.context.format):
raise falcon.HTTPBadRequest(
description="Parameter 'format' must be one of: " +
', '.join(req.context.formatter.list_formats()))
def format_response(self, req: falcon.asgi.Request, resp: falcon.asgi.Response,
result: Any) -> None:
""" Render response into a string according to the formatter
set in `parse_format()`.
"""
resp.text = req.context.formatter.format(result, req.context.format)
resp.content_type = CONTENT_TYPE.get(req.context.format, falcon.MEDIA_JSON)
async def on_get_status(self, req: falcon.asgi.Request, resp: falcon.asgi.Response) -> None:
""" Implementation of status endpoint.
"""
self.parse_format(req, StatusResult, 'text')
result = await self.api.status()
self.format_response(req, resp, result)
if result.status and req.context.format == 'text':
resp.status = 500
await self.func(self.api, ParamWrapper(req, resp))
def get_application(project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> falcon.asgi.App:
""" Create a Nominatim falcon ASGI application.
environ: Optional[Mapping[str, str]] = None) -> App:
""" Create a Nominatim Falcon ASGI application.
"""
app = falcon.asgi.App()
api = NominatimAPIAsync(project_dir, environ)
api = NominatimV1(project_dir, environ)
app = App(cors_enable=api.config.get_bool('CORS_NOACCESSCONTROL'))
app.add_route('/status', api, suffix='status')
legacy_urls = api.config.get_bool('SERVE_LEGACY_URLS')
for name, func in api_impl.ROUTES:
endpoint = EndpointWrapper(func, api)
app.add_route(f"/{name}", endpoint)
if legacy_urls:
app.add_route(f"/{name}.php", endpoint)
return app

94
nominatim/server/sanic/server.py
View File

@@ -2,85 +2,71 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the sanic webserver framework.
"""
from typing import Any, Optional, Mapping
from typing import Any, Optional, Mapping, Callable, cast, Coroutine
from pathlib import Path
import sanic
from sanic import Request, HTTPResponse, Sanic
from sanic.exceptions import SanicException
from sanic.response import text as TextResponse
from nominatim.api import NominatimAPIAsync
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
import nominatim.api.v1 as api_impl
api = sanic.Blueprint('NominatimAPI')
CONTENT_TYPE = {
'text': 'text/plain; charset=utf-8',
'xml': 'text/xml; charset=utf-8'
}
def usage_error(msg: str) -> sanic.HTTPResponse:
""" Format the response for an error with the query parameters.
class ParamWrapper(api_impl.ASGIAdaptor):
""" Adaptor class for server glue to Sanic framework.
"""
return sanic.response.text(msg, status=400)
def __init__(self, request: Request) -> None:
self.request = request
def api_response(request: sanic.Request, result: Any) -> sanic.HTTPResponse:
""" Render a response from the query results using the configured
formatter.
"""
body = request.ctx.formatter.format(result, request.ctx.format)
return sanic.response.text(body,
content_type=CONTENT_TYPE.get(request.ctx.format,
'application/json'))
def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
return cast(Optional[str], self.request.args.get(name, default))
@api.on_request # type: ignore[misc]
async def extract_format(request: sanic.Request) -> Optional[sanic.HTTPResponse]:
""" Get and check the 'format' parameter and prepare the formatter.
`ctx.result_type` describes the expected return type and
`ctx.default_format` the format value to assume when no parameter
is present.
"""
assert request.route is not None
request.ctx.formatter = request.app.ctx.formatters[request.route.ctx.result_type]
request.ctx.format = request.args.get('format', request.route.ctx.default_format)
if not request.ctx.formatter.supports_format(request.ctx.format):
return usage_error("Parameter 'format' must be one of: " +
', '.join(request.ctx.formatter.list_formats()))
return None
def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
return cast(Optional[str], self.request.headers.get(name, default))
@api.get('/status', ctx_result_type=StatusResult, ctx_default_format='text')
async def status(request: sanic.Request) -> sanic.HTTPResponse:
""" Implementation of status endpoint.
"""
result = await request.app.ctx.api.status()
response = api_response(request, result)
def error(self, msg: str) -> SanicException:
return SanicException(msg, status_code=400)
if request.ctx.format == 'text' and result.status:
response.status = 500
return response
def create_response(self, status: int, output: str,
content_type: str) -> HTTPResponse:
return TextResponse(output, status=status, content_type=content_type)
def _wrap_endpoint(func: api_impl.EndpointFunc)\
-> Callable[[Request], Coroutine[Any, Any, HTTPResponse]]:
async def _callback(request: Request) -> HTTPResponse:
return cast(HTTPResponse, await func(request.app.ctx.api, ParamWrapper(request)))
return _callback
def get_application(project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> sanic.Sanic:
environ: Optional[Mapping[str, str]] = None) -> Sanic:
""" Create a Nominatim sanic ASGI application.
"""
app = sanic.Sanic("NominatimInstance")
app = Sanic("NominatimInstance")
app.ctx.api = NominatimAPIAsync(project_dir, environ)
app.ctx.formatters = {}
for rtype in (StatusResult, ):
app.ctx.formatters[rtype] = formatting.create(rtype)
app.blueprint(api)
if app.ctx.api.config.get_bool('CORS_NOACCESSCONTROL'):
from sanic_cors import CORS # pylint: disable=import-outside-toplevel
CORS(app)
legacy_urls = app.ctx.api.config.get_bool('SERVE_LEGACY_URLS')
for name, func in api_impl.ROUTES:
endpoint = _wrap_endpoint(func)
app.add_route(endpoint, f"/{name}", name=f"v1_{name}_simple")
if legacy_urls:
app.add_route(endpoint, f"/{name}.php", name=f"v1_{name}_legacy")
return app

92
nominatim/server/starlette/server.py
View File

@@ -2,12 +2,12 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the starlette webserver framework.
"""
from typing import Any, Type, Optional, Mapping
from typing import Any, Optional, Mapping, Callable, cast, Coroutine
from pathlib import Path
from starlette.applications import Starlette
@@ -15,68 +15,64 @@ from starlette.routing import Route
from starlette.exceptions import HTTPException
from starlette.responses import Response
from starlette.requests import Request
from starlette.middleware import Middleware
from starlette.middleware.cors import CORSMiddleware
from nominatim.config import Configuration
from nominatim.api import NominatimAPIAsync
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
import nominatim.api.v1 as api_impl
CONTENT_TYPE = {
'text': 'text/plain; charset=utf-8',
'xml': 'text/xml; charset=utf-8'
}
FORMATTERS = {
StatusResult: formatting.create(StatusResult)
}
def parse_format(request: Request, rtype: Type[Any], default: str) -> None:
""" Get and check the 'format' parameter and prepare the formatter.
`rtype` describes the expected return type and `default` the
format value to assume when no parameter is present.
class ParamWrapper(api_impl.ASGIAdaptor):
""" Adaptor class for server glue to Starlette framework.
"""
fmt = request.query_params.get('format', default=default)
fmtter = FORMATTERS[rtype]
if not fmtter.supports_format(fmt):
raise HTTPException(400, detail="Parameter 'format' must be one of: " +
', '.join(fmtter.list_formats()))
request.state.format = fmt
request.state.formatter = fmtter
def __init__(self, request: Request) -> None:
self.request = request
def format_response(request: Request, result: Any) -> Response:
""" Render response into a string according to the formatter
set in `parse_format()`.
"""
fmt = request.state.format
return Response(request.state.formatter.format(result, fmt),
media_type=CONTENT_TYPE.get(fmt, 'application/json'))
def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
return self.request.query_params.get(name, default=default)
async def on_status(request: Request) -> Response:
""" Implementation of status endpoint.
"""
parse_format(request, StatusResult, 'text')
result = await request.app.state.API.status()
response = format_response(request, result)
if request.state.format == 'text' and result.status:
response.status_code = 500
return response
def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
return self.request.headers.get(name, default)
V1_ROUTES = [
Route('/status', endpoint=on_status)
]
def error(self, msg: str) -> HTTPException:
return HTTPException(400, detail=msg)
def create_response(self, status: int, output: str, content_type: str) -> Response:
return Response(output, status_code=status, media_type=content_type)
def _wrap_endpoint(func: api_impl.EndpointFunc)\
-> Callable[[Request], Coroutine[Any, Any, Response]]:
async def _callback(request: Request) -> Response:
return cast(Response, await func(request.app.state.API, ParamWrapper(request)))
return _callback
def get_application(project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> Starlette:
""" Create a Nominatim falcon ASGI application.
"""
app = Starlette(debug=True, routes=V1_ROUTES)
config = Configuration(project_dir, environ)
routes = []
legacy_urls = config.get_bool('SERVE_LEGACY_URLS')
for name, func in api_impl.ROUTES:
endpoint = _wrap_endpoint(func)
routes.append(Route(f"/{name}", endpoint=endpoint))
if legacy_urls:
routes.append(Route(f"/{name}.php", endpoint=endpoint))
middleware = []
if config.get_bool('CORS_NOACCESSCONTROL'):
middleware.append(Middleware(CORSMiddleware, allow_origins=['*']))
app = Starlette(debug=True, routes=routes, middleware=middleware)
app.state.API = NominatimAPIAsync(project_dir, environ)

5
settings/env.defaults
View File

@@ -204,6 +204,11 @@ NOMINATIM_LOOKUP_MAX_COUNT=50
# Set to zero to disable polygon output.
NOMINATIM_POLYGON_OUTPUT_MAX_TYPES=1
# Offer backwards compatible PHP URLs.
# When running one of the Python enignes, they will add endpoint aliases
# under <endpoint>.php
NOMINATIM_SERVE_LEGACY_URLS=yes
### Log settings
#
# The following options allow to enable logging of API requests.

57
test/python/api/test_result_formatting_v1.py Normal file
View File

@@ -0,0 +1,57 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Tests for formatting results for the V1 API.
"""
import datetime as dt
import pytest
import nominatim.api.v1 as api_impl
from nominatim.api import StatusResult
from nominatim.version import NOMINATIM_VERSION
STATUS_FORMATS = {'text', 'json'}
# StatusResult
def test_status_format_list():
assert set(api_impl.list_formats(StatusResult)) == STATUS_FORMATS
@pytest.mark.parametrize('fmt', list(STATUS_FORMATS))
def test_status_supported(fmt):
assert api_impl.supports_format(StatusResult, fmt)
def test_status_unsupported():
assert not api_impl.supports_format(StatusResult, 'gagaga')
def test_status_format_text():
assert api_impl.format_result(StatusResult(0, 'message here'), 'text') == 'OK'
def test_status_format_text():
assert api_impl.format_result(StatusResult(500, 'message here'), 'text') == 'ERROR: message here'
def test_status_format_json_minimal():
status = StatusResult(700, 'Bad format.')
result = api_impl.format_result(status, 'json')
assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, )
def test_status_format_json_full():
status = StatusResult(0, 'OK')
status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc)
status.database_version = '5.6'
result = api_impl.format_result(status, 'json')
assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, )

9
test/python/cli/test_cmd_api.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2023 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Tests for API access commands of command-line interface wrapper.
@@ -11,8 +11,7 @@ import json
import pytest
import nominatim.clicmd.api
import nominatim.api
from nominatim.apicmd.status import StatusResult
import nominatim.api as napi
@pytest.mark.parametrize("endpoint", (('search', 'reverse', 'lookup', 'details', 'status')))
@@ -61,8 +60,8 @@ class TestCliStatusCall:
@pytest.fixture(autouse=True)
def setup_status_mock(self, monkeypatch):
monkeypatch.setattr(nominatim.api.NominatimAPI, 'status',
lambda self: StatusResult(200, 'OK'))
monkeypatch.setattr(napi.NominatimAPI, 'status',
lambda self: napi.StatusResult(200, 'OK'))
def test_status_simple(self, cli_call, tmp_path):

63
test/python/result_formatter/test_v1.py
View File

@@ -1,63 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Tests for formatting results for the V1 API.
"""
import datetime as dt
import pytest
import nominatim.result_formatter.v1 as format_module
from nominatim.apicmd.status import StatusResult
from nominatim.version import NOMINATIM_VERSION
STATUS_FORMATS = {'text', 'json'}
class TestStatusResultFormat:
@pytest.fixture(autouse=True)
def make_formatter(self):
self.formatter = format_module.create(StatusResult)
def test_format_list(self):
assert set(self.formatter.list_formats()) == STATUS_FORMATS
@pytest.mark.parametrize('fmt', list(STATUS_FORMATS))
def test_supported(self, fmt):
assert self.formatter.supports_format(fmt)
def test_unsupported(self):
assert not self.formatter.supports_format('gagaga')
def test_format_text(self):
assert self.formatter.format(StatusResult(0, 'message here'), 'text') == 'OK'
def test_format_text(self):
assert self.formatter.format(StatusResult(500, 'message here'), 'text') == 'ERROR: message here'
def test_format_json_minimal(self):
status = StatusResult(700, 'Bad format.')
result = self.formatter.format(status, 'json')
assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, )
def test_format_json_full(self):
status = StatusResult(0, 'OK')
status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc)
status.database_version = '5.6'
result = self.formatter.format(status, 'json')
assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, )