diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 0f4aea26..a4de7149 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -107,7 +107,7 @@ jobs: if: matrix.flavour == 'oldstuff' - name: Install Python webservers - run: pip3 install falcon sanic sanic-testing starlette + run: pip3 install falcon sanic sanic-testing sanic-cors starlette - name: Install latest pylint/mypy run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs diff --git a/.mypy.ini b/.mypy.ini index ef2057d4..611c3c5d 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,6 +1,9 @@ [mypy] plugins = sqlalchemy.ext.mypy.plugin +[mypy-sanic_cors.*] +ignore_missing_imports = True + [mypy-icu.*] ignore_missing_imports = True diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index f6692f58..663d5c37 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -66,7 +66,7 @@ For running the experimental Python frontend: * one of the following web frameworks: * [falcon](https://falconframework.org/) (3.0+) - * [sanic](https://sanic.dev) + * [sanic](https://sanic.dev) and (optionally) [sanic-cors](https://github.com/ashleysommer/sanic-cors) * [starlette](https://www.starlette.io/) * [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks) diff --git a/nominatim/api.py b/nominatim/api.py deleted file mode 100644 index 10cca533..00000000 --- a/nominatim/api.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2022 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Implementation of classes for API access via libraries. -""" -from typing import Mapping, Optional, cast, Any -import asyncio -from pathlib import Path - -from sqlalchemy import text, event -from sqlalchemy.engine.url import URL -from sqlalchemy.ext.asyncio import create_async_engine -import asyncpg - -from nominatim.config import Configuration -from nominatim.apicmd.status import get_status, StatusResult - -class NominatimAPIAsync: - """ API loader asynchornous version. - """ - def __init__(self, project_dir: Path, - environ: Optional[Mapping[str, str]] = None) -> None: - self.config = Configuration(project_dir, environ) - - dsn = self.config.get_database_params() - - dburl = URL.create( - 'postgresql+asyncpg', - database=dsn.get('dbname'), - username=dsn.get('user'), password=dsn.get('password'), - host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None, - query={k: v for k, v in dsn.items() - if k not in ('user', 'password', 'dbname', 'host', 'port')}) - self.engine = create_async_engine( - dburl, future=True, - connect_args={'server_settings': { - 'DateStyle': 'sql,european', - 'max_parallel_workers_per_gather': '0' - }}) - asyncio.get_event_loop().run_until_complete(self._query_server_version()) - asyncio.get_event_loop().run_until_complete(self.close()) - - if self.server_version >= 110000: - @event.listens_for(self.engine.sync_engine, "connect") # type: ignore[misc] - def _on_connect(dbapi_con: Any, _: Any) -> None: - cursor = dbapi_con.cursor() - cursor.execute("SET jit_above_cost TO '-1'") - - - async def _query_server_version(self) -> None: - try: - async with self.engine.begin() as conn: - result = await conn.scalar(text('SHOW server_version_num')) - self.server_version = int(cast(str, result)) - except asyncpg.PostgresError: - self.server_version = 0 - - async def close(self) -> None: - """ Close all active connections to the database. The NominatimAPIAsync - object remains usable after closing. If a new API functions is - called, new connections are created. - """ - await self.engine.dispose() - - - async def status(self) -> StatusResult: - """ Return the status of the database. - """ - return await get_status(self.engine) - - -class NominatimAPI: - """ API loader, synchronous version. - """ - - def __init__(self, project_dir: Path, - environ: Optional[Mapping[str, str]] = None) -> None: - self.async_api = NominatimAPIAsync(project_dir, environ) - - - def close(self) -> None: - """ Close all active connections to the database. The NominatimAPIAsync - object remains usable after closing. If a new API functions is - called, new connections are created. - """ - asyncio.get_event_loop().run_until_complete(self.async_api.close()) - - - def status(self) -> StatusResult: - """ Return the status of the database. - """ - return asyncio.get_event_loop().run_until_complete(self.async_api.status()) diff --git a/nominatim/api/__init__.py b/nominatim/api/__init__.py new file mode 100644 index 00000000..f418e663 --- /dev/null +++ b/nominatim/api/__init__.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +The public interface of the Nominatim library. + +Classes and functions defined in this file are considered stable. Always +import from this file, not from the source files directly. +""" + +# See also https://github.com/PyCQA/pylint/issues/6006 +# pylint: disable=useless-import-alias + +from nominatim.api.core import (NominatimAPI as NominatimAPI, + NominatimAPIAsync as NominatimAPIAsync) +from nominatim.api.status import (StatusResult as StatusResult) diff --git a/nominatim/api/core.py b/nominatim/api/core.py new file mode 100644 index 00000000..159229dd --- /dev/null +++ b/nominatim/api/core.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Implementation of classes for API access via libraries. +""" +from typing import Mapping, Optional, Any, AsyncIterator +import asyncio +import contextlib +from pathlib import Path + +import sqlalchemy as sa +import sqlalchemy.ext.asyncio as sa_asyncio +import asyncpg + +from nominatim.config import Configuration +from nominatim.api.status import get_status, StatusResult + +class NominatimAPIAsync: + """ API loader asynchornous version. + """ + def __init__(self, project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> None: + self.config = Configuration(project_dir, environ) + self.server_version = 0 + + self._engine_lock = asyncio.Lock() + self._engine: Optional[sa_asyncio.AsyncEngine] = None + + + async def setup_database(self) -> None: + """ Set up the engine and connection parameters. + + This function will be implicitly called when the database is + accessed for the first time. You may also call it explicitly to + avoid that the first call is delayed by the setup. + """ + async with self._engine_lock: + if self._engine: + return + + dsn = self.config.get_database_params() + + dburl = sa.engine.URL.create( + 'postgresql+asyncpg', + database=dsn.get('dbname'), + username=dsn.get('user'), password=dsn.get('password'), + host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None, + query={k: v for k, v in dsn.items() + if k not in ('user', 'password', 'dbname', 'host', 'port')}) + engine = sa_asyncio.create_async_engine( + dburl, future=True, + connect_args={'server_settings': { + 'DateStyle': 'sql,european', + 'max_parallel_workers_per_gather': '0' + }}) + + try: + async with engine.begin() as conn: + result = await conn.scalar(sa.text('SHOW server_version_num')) + self.server_version = int(result) + except asyncpg.PostgresError: + self.server_version = 0 + + if self.server_version >= 110000: + @sa.event.listens_for(engine.sync_engine, "connect") # type: ignore[misc] + def _on_connect(dbapi_con: Any, _: Any) -> None: + cursor = dbapi_con.cursor() + cursor.execute("SET jit_above_cost TO '-1'") + # Make sure that all connections get the new settings + await self.close() + + self._engine = engine + + + async def close(self) -> None: + """ Close all active connections to the database. The NominatimAPIAsync + object remains usable after closing. If a new API functions is + called, new connections are created. + """ + if self._engine is not None: + await self._engine.dispose() + + + @contextlib.asynccontextmanager + async def begin(self) -> AsyncIterator[sa_asyncio.AsyncConnection]: + """ Create a new connection with automatic transaction handling. + + This function may be used to get low-level access to the database. + Refer to the documentation of SQLAlchemy for details how to use + the connection object. + """ + if self._engine is None: + await self.setup_database() + + assert self._engine is not None + + async with self._engine.begin() as conn: + yield conn + + + async def status(self) -> StatusResult: + """ Return the status of the database. + """ + try: + async with self.begin() as conn: + status = await get_status(conn) + except asyncpg.PostgresError: + return StatusResult(700, 'Database connection failed') + + return status + + +class NominatimAPI: + """ API loader, synchronous version. + """ + + def __init__(self, project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> None: + self._loop = asyncio.new_event_loop() + self._async_api = NominatimAPIAsync(project_dir, environ) + + + def close(self) -> None: + """ Close all active connections to the database. The NominatimAPIAsync + object remains usable after closing. If a new API functions is + called, new connections are created. + """ + self._loop.run_until_complete(self._async_api.close()) + self._loop.close() + + + def status(self) -> StatusResult: + """ Return the status of the database. + """ + return self._loop.run_until_complete(self._async_api.status()) diff --git a/nominatim/result_formatter/base.py b/nominatim/api/result_formatting.py similarity index 53% rename from nominatim/result_formatter/base.py rename to nominatim/api/result_formatting.py index d77f4db8..09cf7db8 100644 --- a/nominatim/result_formatter/base.py +++ b/nominatim/api/result_formatting.py @@ -2,49 +2,21 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ -Helper classes and function for writing result formatting modules. +Helper classes and functions for formating results into API responses. """ -from typing import Type, TypeVar, Dict, Mapping, List, Callable, Generic, Any +from typing import Type, TypeVar, Dict, List, Callable, Any from collections import defaultdict T = TypeVar('T') # pylint: disable=invalid-name FormatFunc = Callable[[T], str] -class ResultFormatter(Generic[T]): - """ This class dispatches format calls to the appropriate formatting - function previously defined with the `format_func` decorator. - """ - - def __init__(self, funcs: Mapping[str, FormatFunc[T]]) -> None: - self.functions = funcs - - - def list_formats(self) -> List[str]: - """ Return a list of formats supported by this formatter. - """ - return list(self.functions.keys()) - - - def supports_format(self, fmt: str) -> bool: - """ Check if the given format is supported by this formatter. - """ - return fmt in self.functions - - - def format(self, result: T, fmt: str) -> str: - """ Convert the given result into a string using the given format. - - The format is expected to be in the list returned by - `list_formats()`. - """ - return self.functions[fmt](result) - class FormatDispatcher: - """ A factory class for result formatters. + """ Helper class to conveniently create formatting functions in + a module using decorators. """ def __init__(self) -> None: @@ -63,7 +35,22 @@ class FormatDispatcher: return decorator - def __call__(self, result_class: Type[T]) -> ResultFormatter[T]: - """ Create an instance of a format class for the given result type. + def list_formats(self, result_type: Type[Any]) -> List[str]: + """ Return a list of formats supported by this formatter. """ - return ResultFormatter(self.format_functions[result_class]) + return list(self.format_functions[result_type].keys()) + + + def supports_format(self, result_type: Type[Any], fmt: str) -> bool: + """ Check if the given format is supported by this formatter. + """ + return fmt in self.format_functions[result_type] + + + def format_result(self, result: Any, fmt: str) -> str: + """ Convert the given result into a string using the given format. + + The format is expected to be in the list returned by + `list_formats()`. + """ + return self.format_functions[type(result)][fmt](result) diff --git a/nominatim/apicmd/status.py b/nominatim/api/status.py similarity index 69% rename from nominatim/apicmd/status.py rename to nominatim/api/status.py index 85071db9..560953d3 100644 --- a/nominatim/apicmd/status.py +++ b/nominatim/api/status.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Classes and function releated to status call. @@ -10,8 +10,8 @@ Classes and function releated to status call. from typing import Optional, cast import datetime as dt -import sqlalchemy as sqla -from sqlalchemy.ext.asyncio.engine import AsyncEngine, AsyncConnection +import sqlalchemy as sa +from sqlalchemy.ext.asyncio.engine import AsyncConnection import asyncpg from nominatim import version @@ -31,7 +31,7 @@ class StatusResult: async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]: """ Query the database date. """ - sql = sqla.text('SELECT lastimportdate FROM import_status LIMIT 1') + sql = sa.text('SELECT lastimportdate FROM import_status LIMIT 1') result = await conn.execute(sql) for row in result: @@ -41,8 +41,8 @@ async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]: async def _get_database_version(conn: AsyncConnection) -> Optional[version.NominatimVersion]: - sql = sqla.text("""SELECT value FROM nominatim_properties - WHERE property = 'database_version'""") + sql = sa.text("""SELECT value FROM nominatim_properties + WHERE property = 'database_version'""") result = await conn.execute(sql) for row in result: @@ -51,14 +51,13 @@ async def _get_database_version(conn: AsyncConnection) -> Optional[version.Nomin return None -async def get_status(engine: AsyncEngine) -> StatusResult: +async def get_status(conn: AsyncConnection) -> StatusResult: """ Execute a status API call. """ status = StatusResult(0, 'OK') try: - async with engine.begin() as conn: - status.data_updated = await _get_database_date(conn) - status.database_version = await _get_database_version(conn) + status.data_updated = await _get_database_date(conn) + status.database_version = await _get_database_version(conn) except asyncpg.PostgresError: return StatusResult(700, 'Database connection failed') diff --git a/nominatim/api/v1/__init__.py b/nominatim/api/v1/__init__.py new file mode 100644 index 00000000..8c00af2d --- /dev/null +++ b/nominatim/api/v1/__init__.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Implementation of API version v1 (aka the legacy version). +""" + +#pylint: disable=useless-import-alias + +from nominatim.api.v1.server_glue import (ASGIAdaptor as ASGIAdaptor, + EndpointFunc as EndpointFunc, + ROUTES as ROUTES) + +import nominatim.api.v1.format as _format + +list_formats = _format.dispatch.list_formats +supports_format = _format.dispatch.supports_format +format_result = _format.dispatch.format_result diff --git a/nominatim/result_formatter/v1.py b/nominatim/api/v1/format.py similarity index 75% rename from nominatim/result_formatter/v1.py rename to nominatim/api/v1/format.py index 1d437af7..cb2b15a7 100644 --- a/nominatim/result_formatter/v1.py +++ b/nominatim/api/v1/format.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Output formatters for API version v1. @@ -11,12 +11,12 @@ from typing import Dict, Any from collections import OrderedDict import json -from nominatim.result_formatter.base import FormatDispatcher -from nominatim.apicmd.status import StatusResult +from nominatim.api.result_formatting import FormatDispatcher +from nominatim.api import StatusResult -create = FormatDispatcher() +dispatch = FormatDispatcher() -@create.format_func(StatusResult, 'text') +@dispatch.format_func(StatusResult, 'text') def _format_status_text(result: StatusResult) -> str: if result.status: return f"ERROR: {result.message}" @@ -24,7 +24,7 @@ def _format_status_text(result: StatusResult) -> str: return 'OK' -@create.format_func(StatusResult, 'json') +@dispatch.format_func(StatusResult, 'json') def _format_status_json(result: StatusResult) -> str: out: Dict[str, Any] = OrderedDict() out['status'] = result.status diff --git a/nominatim/api/v1/server_glue.py b/nominatim/api/v1/server_glue.py new file mode 100644 index 00000000..7444b7aa --- /dev/null +++ b/nominatim/api/v1/server_glue.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Generic part of the server implementation of the v1 API. +Combine with the scaffolding provided for the various Python ASGI frameworks. +""" +from typing import Optional, Any, Type, Callable +import abc + +import nominatim.api as napi +from nominatim.api.v1.format import dispatch as formatting + +CONTENT_TYPE = { + 'text': 'text/plain; charset=utf-8', + 'xml': 'text/xml; charset=utf-8', + 'jsonp': 'application/javascript' +} + + +class ASGIAdaptor(abc.ABC): + """ Adapter class for the different ASGI frameworks. + Wraps functionality over concrete requests and responses. + """ + + @abc.abstractmethod + def get(self, name: str, default: Optional[str] = None) -> Optional[str]: + """ Return an input parameter as a string. If the parameter was + not provided, return the 'default' value. + """ + + @abc.abstractmethod + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + """ Return a HTTP header parameter as a string. If the parameter was + not provided, return the 'default' value. + """ + + + @abc.abstractmethod + def error(self, msg: str) -> Exception: + """ Construct an appropriate exception from the given error message. + The exception must result in a HTTP 400 error. + """ + + + @abc.abstractmethod + def create_response(self, status: int, output: str, content_type: str) -> Any: + """ Create a response from the given parameters. The result will + be returned by the endpoint functions. The adaptor may also + return None when the response is created internally with some + different means. + + The response must return the HTTP given status code 'status', set + the HTTP content-type headers to the string provided and the + body of the response to 'output'. + """ + + + def build_response(self, output: str, media_type: str, status: int = 200) -> Any: + """ Create a response from the given output. Wraps a JSONP function + around the response, if necessary. + """ + if media_type == 'json' and status == 200: + jsonp = self.get('json_callback') + if jsonp is not None: + if any(not part.isidentifier() for part in jsonp.split('.')): + raise self.error('Invalid json_callback value') + output = f"{jsonp}({output})" + media_type = 'jsonp' + + return self.create_response(status, output, + CONTENT_TYPE.get(media_type, 'application/json')) + + + def get_int(self, name: str, default: Optional[int] = None) -> int: + """ Return an input parameter as an int. Raises an exception if + the parameter is given but not in an integer format. + + If 'default' is given, then it will be returned when the parameter + is missing completely. When 'default' is None, an error will be + raised on a missing parameter. + """ + value = self.get(name) + + if value is None: + if default is not None: + return default + + raise self.error(f"Parameter '{name}' missing.") + + try: + return int(value) + except ValueError as exc: + raise self.error(f"Parameter '{name}' must be a number.") from exc + + + def get_bool(self, name: str, default: Optional[bool] = None) -> bool: + """ Return an input parameter as bool. Only '0' is accepted as + an input for 'false' all other inputs will be interpreted as 'true'. + + If 'default' is given, then it will be returned when the parameter + is missing completely. When 'default' is None, an error will be + raised on a missing parameter. + """ + value = self.get(name) + + if value is None: + if default is not None: + return default + + raise self.error(f"Parameter '{name}' missing.") + + return value != '0' + + +def parse_format(params: ASGIAdaptor, result_type: Type[Any], default: str) -> str: + """ Get and check the 'format' parameter and prepare the formatter. + `fmtter` is a formatter and `default` the + format value to assume when no parameter is present. + """ + fmt = params.get('format', default=default) + assert fmt is not None + + if not formatting.supports_format(result_type, fmt): + raise params.error("Parameter 'format' must be one of: " + + ', '.join(formatting.list_formats(result_type))) + + return fmt + + +async def status_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: + """ Server glue for /status endpoint. See API docs for details. + """ + result = await api.status() + + fmt = parse_format(params, napi.StatusResult, 'text') + + if fmt == 'text' and result.status: + status_code = 500 + else: + status_code = 200 + + return params.build_response(formatting.format_result(result, fmt), fmt, + status=status_code) + +EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any] + +ROUTES = [ + ('status', status_endpoint) +] diff --git a/nominatim/apicmd/__init__.py b/nominatim/apicmd/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/nominatim/cli.py b/nominatim/cli.py index cedbdb4a..d34ef118 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -236,7 +236,7 @@ class AdminServe: server_module = importlib.import_module('nominatim.server.sanic.server') app = server_module.get_application(args.project_dir) - app.run(host=host, port=port, debug=True) + app.run(host=host, port=port, debug=True, single_process=True) else: import uvicorn # pylint: disable=import-outside-toplevel diff --git a/nominatim/clicmd/api.py b/nominatim/clicmd/api.py index e2b903c7..cc65f5f6 100644 --- a/nominatim/clicmd/api.py +++ b/nominatim/clicmd/api.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Subcommand definitions for API calls from the command line. @@ -14,9 +14,8 @@ import logging from nominatim.tools.exec_utils import run_api_script from nominatim.errors import UsageError from nominatim.clicmd.args import NominatimArgs -from nominatim.api import NominatimAPI -from nominatim.apicmd.status import StatusResult -import nominatim.result_formatter.v1 as formatting +from nominatim.api import NominatimAPI, StatusResult +import nominatim.api.v1 as api_output # Do not repeat documentation of subcommand classes. # pylint: disable=C0111 @@ -277,7 +276,7 @@ class APIStatus: """ def add_args(self, parser: argparse.ArgumentParser) -> None: - formats = formatting.create(StatusResult).list_formats() + formats = api_output.list_formats(StatusResult) group = parser.add_argument_group('API parameters') group.add_argument('--format', default=formats[0], choices=formats, help='Format of result') @@ -285,5 +284,5 @@ class APIStatus: def run(self, args: NominatimArgs) -> int: status = NominatimAPI(args.project_dir).status() - print(formatting.create(StatusResult).format(status, args.format)) + print(api_output.format_result(status, args.format)) return 0 diff --git a/nominatim/result_formatter/__init__.py b/nominatim/result_formatter/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/nominatim/server/falcon/server.py b/nominatim/server/falcon/server.py index 81e6ed39..080650e7 100644 --- a/nominatim/server/falcon/server.py +++ b/nominatim/server/falcon/server.py @@ -2,81 +2,76 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Server implementation using the falcon webserver framework. """ -from typing import Type, Any, Optional, Mapping +from typing import Optional, Mapping, cast from pathlib import Path import falcon -import falcon.asgi +from falcon.asgi import App, Request, Response from nominatim.api import NominatimAPIAsync -from nominatim.apicmd.status import StatusResult -import nominatim.result_formatter.v1 as formatting +import nominatim.api.v1 as api_impl -CONTENT_TYPE = { - 'text': falcon.MEDIA_TEXT, - 'xml': falcon.MEDIA_XML -} -class NominatimV1: - """ Implementation of V1 version of the Nominatim API. +class ParamWrapper(api_impl.ASGIAdaptor): + """ Adaptor class for server glue to Falcon framework. """ - def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]]) -> None: - self.api = NominatimAPIAsync(project_dir, environ) - self.formatters = {} - - for rtype in (StatusResult, ): - self.formatters[rtype] = formatting.create(rtype) + def __init__(self, req: Request, resp: Response) -> None: + self.request = req + self.response = resp - def parse_format(self, req: falcon.asgi.Request, rtype: Type[Any], default: str) -> None: - """ Get and check the 'format' parameter and prepare the formatter. - `rtype` describes the expected return type and `default` the - format value to assume when no parameter is present. + def get(self, name: str, default: Optional[str] = None) -> Optional[str]: + return cast(Optional[str], self.request.get_param(name, default=default)) + + + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + return cast(Optional[str], self.request.get_header(name, default=default)) + + + def error(self, msg: str) -> falcon.HTTPBadRequest: + return falcon.HTTPBadRequest(description=msg) + + + def create_response(self, status: int, output: str, content_type: str) -> None: + self.response.status = status + self.response.text = output + self.response.content_type = content_type + + +class EndpointWrapper: + """ Converter for server glue endpoint functions to Falcon request handlers. + """ + + def __init__(self, func: api_impl.EndpointFunc, api: NominatimAPIAsync) -> None: + self.func = func + self.api = api + + + async def on_get(self, req: Request, resp: Response) -> None: + """ Implementation of the endpoint. """ - req.context.format = req.get_param('format', default=default) - req.context.formatter = self.formatters[rtype] - - if not req.context.formatter.supports_format(req.context.format): - raise falcon.HTTPBadRequest( - description="Parameter 'format' must be one of: " + - ', '.join(req.context.formatter.list_formats())) - - - def format_response(self, req: falcon.asgi.Request, resp: falcon.asgi.Response, - result: Any) -> None: - """ Render response into a string according to the formatter - set in `parse_format()`. - """ - resp.text = req.context.formatter.format(result, req.context.format) - resp.content_type = CONTENT_TYPE.get(req.context.format, falcon.MEDIA_JSON) - - - async def on_get_status(self, req: falcon.asgi.Request, resp: falcon.asgi.Response) -> None: - """ Implementation of status endpoint. - """ - self.parse_format(req, StatusResult, 'text') - - result = await self.api.status() - - self.format_response(req, resp, result) - if result.status and req.context.format == 'text': - resp.status = 500 + await self.func(self.api, ParamWrapper(req, resp)) def get_application(project_dir: Path, - environ: Optional[Mapping[str, str]] = None) -> falcon.asgi.App: - """ Create a Nominatim falcon ASGI application. + environ: Optional[Mapping[str, str]] = None) -> App: + """ Create a Nominatim Falcon ASGI application. """ - app = falcon.asgi.App() + api = NominatimAPIAsync(project_dir, environ) - api = NominatimV1(project_dir, environ) + app = App(cors_enable=api.config.get_bool('CORS_NOACCESSCONTROL')) - app.add_route('/status', api, suffix='status') + legacy_urls = api.config.get_bool('SERVE_LEGACY_URLS') + for name, func in api_impl.ROUTES: + endpoint = EndpointWrapper(func, api) + app.add_route(f"/{name}", endpoint) + if legacy_urls: + app.add_route(f"/{name}.php", endpoint) return app diff --git a/nominatim/server/sanic/server.py b/nominatim/server/sanic/server.py index 74841f3f..81d62faf 100644 --- a/nominatim/server/sanic/server.py +++ b/nominatim/server/sanic/server.py @@ -2,85 +2,71 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Server implementation using the sanic webserver framework. """ -from typing import Any, Optional, Mapping +from typing import Any, Optional, Mapping, Callable, cast, Coroutine from pathlib import Path -import sanic +from sanic import Request, HTTPResponse, Sanic +from sanic.exceptions import SanicException +from sanic.response import text as TextResponse from nominatim.api import NominatimAPIAsync -from nominatim.apicmd.status import StatusResult -import nominatim.result_formatter.v1 as formatting +import nominatim.api.v1 as api_impl -api = sanic.Blueprint('NominatimAPI') - -CONTENT_TYPE = { - 'text': 'text/plain; charset=utf-8', - 'xml': 'text/xml; charset=utf-8' -} - -def usage_error(msg: str) -> sanic.HTTPResponse: - """ Format the response for an error with the query parameters. +class ParamWrapper(api_impl.ASGIAdaptor): + """ Adaptor class for server glue to Sanic framework. """ - return sanic.response.text(msg, status=400) + + def __init__(self, request: Request) -> None: + self.request = request -def api_response(request: sanic.Request, result: Any) -> sanic.HTTPResponse: - """ Render a response from the query results using the configured - formatter. - """ - body = request.ctx.formatter.format(result, request.ctx.format) - return sanic.response.text(body, - content_type=CONTENT_TYPE.get(request.ctx.format, - 'application/json')) + def get(self, name: str, default: Optional[str] = None) -> Optional[str]: + return cast(Optional[str], self.request.args.get(name, default)) -@api.on_request # type: ignore[misc] -async def extract_format(request: sanic.Request) -> Optional[sanic.HTTPResponse]: - """ Get and check the 'format' parameter and prepare the formatter. - `ctx.result_type` describes the expected return type and - `ctx.default_format` the format value to assume when no parameter - is present. - """ - assert request.route is not None - request.ctx.formatter = request.app.ctx.formatters[request.route.ctx.result_type] - - request.ctx.format = request.args.get('format', request.route.ctx.default_format) - if not request.ctx.formatter.supports_format(request.ctx.format): - return usage_error("Parameter 'format' must be one of: " + - ', '.join(request.ctx.formatter.list_formats())) - - return None + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + return cast(Optional[str], self.request.headers.get(name, default)) -@api.get('/status', ctx_result_type=StatusResult, ctx_default_format='text') -async def status(request: sanic.Request) -> sanic.HTTPResponse: - """ Implementation of status endpoint. - """ - result = await request.app.ctx.api.status() - response = api_response(request, result) + def error(self, msg: str) -> SanicException: + return SanicException(msg, status_code=400) - if request.ctx.format == 'text' and result.status: - response.status = 500 - return response + def create_response(self, status: int, output: str, + content_type: str) -> HTTPResponse: + return TextResponse(output, status=status, content_type=content_type) + + +def _wrap_endpoint(func: api_impl.EndpointFunc)\ + -> Callable[[Request], Coroutine[Any, Any, HTTPResponse]]: + async def _callback(request: Request) -> HTTPResponse: + return cast(HTTPResponse, await func(request.app.ctx.api, ParamWrapper(request))) + + return _callback def get_application(project_dir: Path, - environ: Optional[Mapping[str, str]] = None) -> sanic.Sanic: + environ: Optional[Mapping[str, str]] = None) -> Sanic: """ Create a Nominatim sanic ASGI application. """ - app = sanic.Sanic("NominatimInstance") + app = Sanic("NominatimInstance") app.ctx.api = NominatimAPIAsync(project_dir, environ) - app.ctx.formatters = {} - for rtype in (StatusResult, ): - app.ctx.formatters[rtype] = formatting.create(rtype) - app.blueprint(api) + if app.ctx.api.config.get_bool('CORS_NOACCESSCONTROL'): + from sanic_cors import CORS # pylint: disable=import-outside-toplevel + CORS(app) + + legacy_urls = app.ctx.api.config.get_bool('SERVE_LEGACY_URLS') + for name, func in api_impl.ROUTES: + endpoint = _wrap_endpoint(func) + app.add_route(endpoint, f"/{name}", name=f"v1_{name}_simple") + if legacy_urls: + app.add_route(endpoint, f"/{name}.php", name=f"v1_{name}_legacy") return app diff --git a/nominatim/server/starlette/server.py b/nominatim/server/starlette/server.py index 41ad899c..de9a3f87 100644 --- a/nominatim/server/starlette/server.py +++ b/nominatim/server/starlette/server.py @@ -2,12 +2,12 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Server implementation using the starlette webserver framework. """ -from typing import Any, Type, Optional, Mapping +from typing import Any, Optional, Mapping, Callable, cast, Coroutine from pathlib import Path from starlette.applications import Starlette @@ -15,68 +15,64 @@ from starlette.routing import Route from starlette.exceptions import HTTPException from starlette.responses import Response from starlette.requests import Request +from starlette.middleware import Middleware +from starlette.middleware.cors import CORSMiddleware +from nominatim.config import Configuration from nominatim.api import NominatimAPIAsync -from nominatim.apicmd.status import StatusResult -import nominatim.result_formatter.v1 as formatting +import nominatim.api.v1 as api_impl -CONTENT_TYPE = { - 'text': 'text/plain; charset=utf-8', - 'xml': 'text/xml; charset=utf-8' -} - -FORMATTERS = { - StatusResult: formatting.create(StatusResult) -} - - -def parse_format(request: Request, rtype: Type[Any], default: str) -> None: - """ Get and check the 'format' parameter and prepare the formatter. - `rtype` describes the expected return type and `default` the - format value to assume when no parameter is present. +class ParamWrapper(api_impl.ASGIAdaptor): + """ Adaptor class for server glue to Starlette framework. """ - fmt = request.query_params.get('format', default=default) - fmtter = FORMATTERS[rtype] - if not fmtter.supports_format(fmt): - raise HTTPException(400, detail="Parameter 'format' must be one of: " + - ', '.join(fmtter.list_formats())) - - request.state.format = fmt - request.state.formatter = fmtter + def __init__(self, request: Request) -> None: + self.request = request -def format_response(request: Request, result: Any) -> Response: - """ Render response into a string according to the formatter - set in `parse_format()`. - """ - fmt = request.state.format - return Response(request.state.formatter.format(result, fmt), - media_type=CONTENT_TYPE.get(fmt, 'application/json')) + def get(self, name: str, default: Optional[str] = None) -> Optional[str]: + return self.request.query_params.get(name, default=default) -async def on_status(request: Request) -> Response: - """ Implementation of status endpoint. - """ - parse_format(request, StatusResult, 'text') - result = await request.app.state.API.status() - response = format_response(request, result) - - if request.state.format == 'text' and result.status: - response.status_code = 500 - - return response + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + return self.request.headers.get(name, default) -V1_ROUTES = [ - Route('/status', endpoint=on_status) -] + def error(self, msg: str) -> HTTPException: + return HTTPException(400, detail=msg) + + + def create_response(self, status: int, output: str, content_type: str) -> Response: + return Response(output, status_code=status, media_type=content_type) + + +def _wrap_endpoint(func: api_impl.EndpointFunc)\ + -> Callable[[Request], Coroutine[Any, Any, Response]]: + async def _callback(request: Request) -> Response: + return cast(Response, await func(request.app.state.API, ParamWrapper(request))) + + return _callback + def get_application(project_dir: Path, environ: Optional[Mapping[str, str]] = None) -> Starlette: """ Create a Nominatim falcon ASGI application. """ - app = Starlette(debug=True, routes=V1_ROUTES) + config = Configuration(project_dir, environ) + + routes = [] + legacy_urls = config.get_bool('SERVE_LEGACY_URLS') + for name, func in api_impl.ROUTES: + endpoint = _wrap_endpoint(func) + routes.append(Route(f"/{name}", endpoint=endpoint)) + if legacy_urls: + routes.append(Route(f"/{name}.php", endpoint=endpoint)) + + middleware = [] + if config.get_bool('CORS_NOACCESSCONTROL'): + middleware.append(Middleware(CORSMiddleware, allow_origins=['*'])) + + app = Starlette(debug=True, routes=routes, middleware=middleware) app.state.API = NominatimAPIAsync(project_dir, environ) diff --git a/settings/env.defaults b/settings/env.defaults index 3115f438..84cd24f1 100644 --- a/settings/env.defaults +++ b/settings/env.defaults @@ -204,6 +204,11 @@ NOMINATIM_LOOKUP_MAX_COUNT=50 # Set to zero to disable polygon output. NOMINATIM_POLYGON_OUTPUT_MAX_TYPES=1 +# Offer backwards compatible PHP URLs. +# When running one of the Python enignes, they will add endpoint aliases +# under .php +NOMINATIM_SERVE_LEGACY_URLS=yes + ### Log settings # # The following options allow to enable logging of API requests. diff --git a/test/python/api/test_result_formatting_v1.py b/test/python/api/test_result_formatting_v1.py new file mode 100644 index 00000000..95472916 --- /dev/null +++ b/test/python/api/test_result_formatting_v1.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Tests for formatting results for the V1 API. +""" +import datetime as dt +import pytest + +import nominatim.api.v1 as api_impl +from nominatim.api import StatusResult +from nominatim.version import NOMINATIM_VERSION + +STATUS_FORMATS = {'text', 'json'} + +# StatusResult + +def test_status_format_list(): + assert set(api_impl.list_formats(StatusResult)) == STATUS_FORMATS + + +@pytest.mark.parametrize('fmt', list(STATUS_FORMATS)) +def test_status_supported(fmt): + assert api_impl.supports_format(StatusResult, fmt) + + +def test_status_unsupported(): + assert not api_impl.supports_format(StatusResult, 'gagaga') + + +def test_status_format_text(): + assert api_impl.format_result(StatusResult(0, 'message here'), 'text') == 'OK' + + +def test_status_format_text(): + assert api_impl.format_result(StatusResult(500, 'message here'), 'text') == 'ERROR: message here' + + +def test_status_format_json_minimal(): + status = StatusResult(700, 'Bad format.') + + result = api_impl.format_result(status, 'json') + + assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, ) + + +def test_status_format_json_full(): + status = StatusResult(0, 'OK') + status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc) + status.database_version = '5.6' + + result = api_impl.format_result(status, 'json') + + assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, ) diff --git a/test/python/cli/test_cmd_api.py b/test/python/cli/test_cmd_api.py index 4031441f..b0c2411f 100644 --- a/test/python/cli/test_cmd_api.py +++ b/test/python/cli/test_cmd_api.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2023 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for API access commands of command-line interface wrapper. @@ -11,8 +11,7 @@ import json import pytest import nominatim.clicmd.api -import nominatim.api -from nominatim.apicmd.status import StatusResult +import nominatim.api as napi @pytest.mark.parametrize("endpoint", (('search', 'reverse', 'lookup', 'details', 'status'))) @@ -61,8 +60,8 @@ class TestCliStatusCall: @pytest.fixture(autouse=True) def setup_status_mock(self, monkeypatch): - monkeypatch.setattr(nominatim.api.NominatimAPI, 'status', - lambda self: StatusResult(200, 'OK')) + monkeypatch.setattr(napi.NominatimAPI, 'status', + lambda self: napi.StatusResult(200, 'OK')) def test_status_simple(self, cli_call, tmp_path): diff --git a/test/python/result_formatter/test_v1.py b/test/python/result_formatter/test_v1.py deleted file mode 100644 index 919f5b80..00000000 --- a/test/python/result_formatter/test_v1.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2022 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Tests for formatting results for the V1 API. -""" -import datetime as dt -import pytest - -import nominatim.result_formatter.v1 as format_module -from nominatim.apicmd.status import StatusResult -from nominatim.version import NOMINATIM_VERSION - -STATUS_FORMATS = {'text', 'json'} - -class TestStatusResultFormat: - - - @pytest.fixture(autouse=True) - def make_formatter(self): - self.formatter = format_module.create(StatusResult) - - - def test_format_list(self): - assert set(self.formatter.list_formats()) == STATUS_FORMATS - - - @pytest.mark.parametrize('fmt', list(STATUS_FORMATS)) - def test_supported(self, fmt): - assert self.formatter.supports_format(fmt) - - - def test_unsupported(self): - assert not self.formatter.supports_format('gagaga') - - - def test_format_text(self): - assert self.formatter.format(StatusResult(0, 'message here'), 'text') == 'OK' - - - def test_format_text(self): - assert self.formatter.format(StatusResult(500, 'message here'), 'text') == 'ERROR: message here' - - - def test_format_json_minimal(self): - status = StatusResult(700, 'Bad format.') - - result = self.formatter.format(status, 'json') - - assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, ) - - - def test_format_json_full(self): - status = StatusResult(0, 'OK') - status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc) - status.database_version = '5.6' - - result = self.formatter.format(status, 'json') - - assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, )