safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

Merge pull request #2937 from lonvia/python-server-stub

Browse Source 
Scaffolding for new Python-based search frontend
This commit is contained in:
Sarah Hoffmann
2023-01-03 14:26:33 +01:00
committed by GitHub
parent 90b29aa808 31d0468cd2
commit bf1f6a997c
43 changed files with 990 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/actions/build-nominatim/action.yml vendored
View File

@@ -27,9 +27,10 @@ runs:
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
if [ "x$UBUNTUVER" == "x18" ]; then
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 datrie asyncpg
else
sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg
pip3 install sqlalchemy
fi
shell: bash
env:

79
.github/workflows/ci-tests.yml vendored
View File

@@ -42,17 +42,14 @@ jobs:
- ubuntu: 18
postgresql: 9.6
postgis: 2.5
pytest: pytest
php: 7.2
- ubuntu: 20
postgresql: 13
postgis: 3
pytest: py.test-3
php: 7.4
- ubuntu: 22
postgresql: 15
postgis: 3
pytest: py.test-3
php: 8.1
runs-on: ubuntu-${{ matrix.ubuntu }}.04
@@ -74,7 +71,7 @@ jobs:
- uses: actions/setup-python@v4
with:
python-version: 3.6
python-version: 3.7
if: matrix.ubuntu == 18
- uses: ./Nominatim/.github/actions/setup-postgresql
@@ -86,50 +83,58 @@ jobs:
with:
ubuntu: ${{ matrix.ubuntu }}
- name: Install test prerequsites
run: sudo apt-get install -y -qq python3-pytest python3-behave
- name: Install test prerequsites (behave from apt)
run: sudo apt-get install -y -qq python3-behave
if: matrix.ubuntu == 20
- name: Install test prerequsites
run: pip3 install pylint pytest behave==1.2.6
- name: Install test prerequsites (behave from pip)
run: pip3 install behave==1.2.6
if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }}
- name: Install test prerequsites
run: sudo apt-get install -y -qq python3-pytest
if: matrix.ubuntu == 22
- name: Install test prerequsites (from apt for Ununtu 2x)
run: sudo apt-get install -y -qq python3-pytest uvicorn
if: matrix.ubuntu >= 20
- name: Install test prerequsites (from pip for Ubuntu 18)
run: pip3 install pytest uvicorn
if: matrix.ubuntu == 18
- name: Install Python webservers
run: pip3 install falcon sanic sanic-testing starlette
- name: Install latest pylint/mypy
run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions
run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs
- name: PHP linting
run: phpcs --report-width=120 .
working-directory: Nominatim
- name: Python linting
run: pylint nominatim
run: python3 -m pylint nominatim
working-directory: Nominatim
- name: Python static typechecking
run: mypy --strict nominatim
working-directory: Nominatim
- name: PHP unit tests
run: phpunit ./
working-directory: Nominatim/test/php
if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }}
- name: Python unit tests
run: $PYTEST test/python
run: python3 -m pytest test/python
working-directory: Nominatim
env:
PYTEST: ${{ matrix.pytest }}
- name: BDD tests
run: |
behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
working-directory: Nominatim/test/bdd
- name: Install newer Python packages (for typechecking info)
run: pip3 install -U osmium uvicorn
if: matrix.ubuntu >= 20
- name: Python static typechecking
run: python3 -m mypy --strict nominatim
working-directory: Nominatim
if: matrix.ubuntu >= 20
legacy-test:
needs: create-archive
@@ -166,7 +171,7 @@ jobs:
- name: BDD tests (legacy tokenizer)
run: |
behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
working-directory: Nominatim/test/bdd
@@ -176,20 +181,13 @@ jobs:
strategy:
matrix:
name: [Ubuntu-18, Ubuntu-20, Ubuntu-22]
name: [Ubuntu-20, Ubuntu-22]
include:
- name: Ubuntu-18
flavour: ubuntu
image: "ubuntu:18.04"
ubuntu: 18
install_mode: install-nginx
- name: Ubuntu-20
flavour: ubuntu
image: "ubuntu:20.04"
ubuntu: 20
install_mode: install-apache
- name: Ubuntu-22
flavour: ubuntu
image: "ubuntu:22.04"
ubuntu: 22
install_mode: install-apache
@@ -212,14 +210,6 @@ jobs:
apt-get install -y git sudo wget
ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
shell: bash
if: matrix.flavour == 'ubuntu'
- name: Prepare container (CentOS)
run: |
dnf update -y
dnf install -y sudo glibc-langpack-en
shell: bash
if: matrix.flavour == 'centos'
- name: Setup import user
run: |
@@ -253,14 +243,6 @@ jobs:
mkdir data-env-reverse
working-directory: /home/nominatim
- name: Prepare import environment (CentOS)
run: |
sudo ln -s /usr/local/bin/nominatim /usr/bin/nominatim
echo NOMINATIM_DATABASE_WEBUSER="apache" > nominatim-project/.env
cp nominatim-project/.env data-env-reverse/.env
working-directory: /home/nominatim
if: matrix.flavour == 'centos'
- name: Print version
run: nominatim --version
working-directory: /home/nominatim/nominatim-project
@@ -268,7 +250,7 @@ jobs:
- name: Collect host OS information
run: nominatim admin --collect-os-info
working-directory: /home/nominatim/nominatim-project
- name: Import
run: nominatim import --osm-file ../test.pbf
working-directory: /home/nominatim/nominatim-project
@@ -288,7 +270,6 @@ jobs:
- name: Prepare update (Ubuntu)
run: apt-get install -y python3-pip
shell: bash
if: matrix.flavour == 'ubuntu'
- name: Run update
run: |

6
.mypy.ini
View File

@@ -1,9 +1,10 @@
[mypy]
plugins = sqlalchemy.ext.mypy.plugin
[mypy-icu.*]
ignore_missing_imports = True
[mypy-osmium.*]
[mypy-asyncpg.*]
ignore_missing_imports = True
[mypy-datrie.*]
@@ -11,3 +12,6 @@ ignore_missing_imports = True
[mypy-dotenv.*]
ignore_missing_imports = True
[mypy-falcon.*]
ignore_missing_imports = True

2
.pylintrc
View File

@@ -1,6 +1,6 @@
[MASTER]
extension-pkg-whitelist=osmium
extension-pkg-whitelist=osmium,falcon
ignored-modules=icu,datrie
[MESSAGES CONTROL]

2
CMakeLists.txt
View File

@@ -73,7 +73,7 @@ endif()
#-----------------------------------------------------------------------------
if (BUILD_IMPORTER)
find_package(PythonInterp 3.6 REQUIRED)
find_package(PythonInterp 3.7 REQUIRED)
endif()
#-----------------------------------------------------------------------------

1
docs/CMakeLists.txt
View File

@@ -23,7 +23,6 @@ foreach (src ${DOC_SOURCES})
endforeach()
ADD_CUSTOM_TARGET(doc
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md
COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml

13
docs/admin/Installation.md
View File

@@ -6,7 +6,6 @@ the following operating systems:
* [Ubuntu 22.04](../appendix/Install-on-Ubuntu-22.md)
* [Ubuntu 20.04](../appendix/Install-on-Ubuntu-20.md)
* [Ubuntu 18.04](../appendix/Install-on-Ubuntu-18.md)
These OS-specific instructions can also be found in executable form
in the `vagrant/` directory.
@@ -44,11 +43,13 @@ For running Nominatim:
* [PostgreSQL](https://www.postgresql.org) (9.6+ will work, 11+ strongly recommended)
* [PostGIS](https://postgis.net) (2.2+ will work, 3.0+ strongly recommended)
* [Python 3](https://www.python.org/) (3.6+)
* [Python 3](https://www.python.org/) (3.7+)
* [Psycopg2](https://www.psycopg.org) (2.7+)
* [Python Dotenv](https://github.com/theskumar/python-dotenv)
* [psutil](https://github.com/giampaolo/psutil)
* [Jinja2](https://palletsprojects.com/p/jinja/)
* [SQLAlchemy](https://www.sqlalchemy.org/) (1.4+ with greenlet support)
* [asyncpg](https://magicstack.github.io/asyncpg) (0.8+)
* [PyICU](https://pypi.org/project/PyICU/)
* [PyYaml](https://pyyaml.org/) (5.1+)
* [datrie](https://github.com/pytries/datrie)
@@ -61,6 +62,14 @@ For running continuous updates:
* [pyosmium](https://osmcode.org/pyosmium/)
For running the experimental Python frontend:
* one of the following web frameworks:
* [falcon](https://falconframework.org/) (3.0+)
* [sanic](https://sanic.dev)
* [starlette](https://www.starlette.io/)
* [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks)
For dependencies for running tests and building documentation, see
the [Development section](../develop/Development-Environment.md).

10
docs/develop/Development-Environment.md
View File

@@ -37,6 +37,13 @@ It has the following additional requirements:
* [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9)
* [pytest](https://pytest.org)
For testing the Python search frontend, you need to install extra dependencies
depending on your choice of webserver framework:
* [sanic-testing](https://sanic.dev/en/plugins/sanic-testing/getting-started.html) (sanic only)
* [httpx](https://www.python-httpx.org/) (starlette only)
* [asgi-lifespan](https://github.com/florimondmanca/asgi-lifespan) (starlette only)
The documentation is built with mkdocs:
* [mkdocs](https://www.mkdocs.org/) >= 1.1.2
@@ -56,7 +63,8 @@ sudo apt install php-cgi phpunit php-codesniffer \
python3-pip python3-setuptools python3-dev
pip3 install --user behave mkdocs mkdocstrings pytest pylint \
mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil
mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \
sanic-testing httpx asgi-lifespan
```
The `mkdocs` executable will be located in `.local/bin`. You may have to add

8
docs/develop/Testing.md
View File

@@ -84,6 +84,8 @@ The tests can be configured with a set of environment variables (`behave -D key=
* `TEST_DB` - name of test database (db tests)
* `API_TEST_DB` - name of the database containing the API test data (api tests)
* `API_TEST_FILE` - OSM file to be imported into the API test database (api tests)
* `API_ENGINE` - webframe to use for running search queries, same values as
`nominatim serve --engine` parameter
* `DB_HOST` - (optional) hostname of database host
* `DB_PORT` - (optional) port of database on host
* `DB_USER` - (optional) username of database login
@@ -120,7 +122,7 @@ and compromises the following data:
API tests should only be testing the functionality of the website PHP code.
Most tests should be formulated as BDD DB creation tests (see below) instead.
#### Code Coverage
#### Code Coverage (PHP engine only)
The API tests also support code coverage tests. You need to install
[PHP_CodeCoverage](https://github.com/sebastianbergmann/php-code-coverage).
@@ -153,7 +155,3 @@ needs superuser rights for postgres.
These tests check that data is imported correctly into the place table. They
use the same template database as the DB Creation tests, so the same remarks apply.
Note that most testing of the gazetteer output of osm2pgsql is done in the tests
of osm2pgsql itself. The BDD tests are just there to ensure compatibility of
the osm2pgsql and Nominatim code.

96
nominatim/api.py Normal file
View File

@@ -0,0 +1,96 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Implementation of classes for API access via libraries.
"""
from typing import Mapping, Optional, cast, Any
import asyncio
from pathlib import Path
from sqlalchemy import text, event
from sqlalchemy.engine.url import URL
from sqlalchemy.ext.asyncio import create_async_engine
import asyncpg
from nominatim.config import Configuration
from nominatim.apicmd.status import get_status, StatusResult
class NominatimAPIAsync:
""" API loader asynchornous version.
"""
def __init__(self, project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> None:
self.config = Configuration(project_dir, environ)
dsn = self.config.get_database_params()
dburl = URL.create(
'postgresql+asyncpg',
database=dsn.get('dbname'),
username=dsn.get('user'), password=dsn.get('password'),
host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None,
query={k: v for k, v in dsn.items()
if k not in ('user', 'password', 'dbname', 'host', 'port')})
self.engine = create_async_engine(
dburl, future=True,
connect_args={'server_settings': {
'DateStyle': 'sql,european',
'max_parallel_workers_per_gather': '0'
}})
asyncio.get_event_loop().run_until_complete(self._query_server_version())
asyncio.get_event_loop().run_until_complete(self.close())
if self.server_version >= 110000:
@event.listens_for(self.engine.sync_engine, "connect") # type: ignore[misc]
def _on_connect(dbapi_con: Any, _: Any) -> None:
cursor = dbapi_con.cursor()
cursor.execute("SET jit_above_cost TO '-1'")
async def _query_server_version(self) -> None:
try:
async with self.engine.begin() as conn:
result = await conn.scalar(text('SHOW server_version_num'))
self.server_version = int(cast(str, result))
except asyncpg.PostgresError:
self.server_version = 0
async def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
called, new connections are created.
"""
await self.engine.dispose()
async def status(self) -> StatusResult:
""" Return the status of the database.
"""
return await get_status(self.engine)
class NominatimAPI:
""" API loader, synchronous version.
"""
def __init__(self, project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> None:
self.async_api = NominatimAPIAsync(project_dir, environ)
def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
called, new connections are created.
"""
asyncio.get_event_loop().run_until_complete(self.async_api.close())
def status(self) -> StatusResult:
""" Return the status of the database.
"""
return asyncio.get_event_loop().run_until_complete(self.async_api.status())

0
nominatim/apicmd/__init__.py Normal file
View File

65
nominatim/apicmd/status.py Normal file
View File

@@ -0,0 +1,65 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Classes and function releated to status call.
"""
from typing import Optional, cast
import datetime as dt
import sqlalchemy as sqla
from sqlalchemy.ext.asyncio.engine import AsyncEngine, AsyncConnection
import asyncpg
from nominatim import version
class StatusResult:
""" Result of a call to the status API.
"""
def __init__(self, status: int, msg: str):
self.status = status
self.message = msg
self.software_version = version.NOMINATIM_VERSION
self.data_updated: Optional[dt.datetime] = None
self.database_version: Optional[version.NominatimVersion] = None
async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]:
""" Query the database date.
"""
sql = sqla.text('SELECT lastimportdate FROM import_status LIMIT 1')
result = await conn.execute(sql)
for row in result:
return cast(dt.datetime, row[0])
return None
async def _get_database_version(conn: AsyncConnection) -> Optional[version.NominatimVersion]:
sql = sqla.text("""SELECT value FROM nominatim_properties
WHERE property = 'database_version'""")
result = await conn.execute(sql)
for row in result:
return version.parse_version(cast(str, row[0]))
return None
async def get_status(engine: AsyncEngine) -> StatusResult:
""" Execute a status API call.
"""
status = StatusResult(0, 'OK')
try:
async with engine.begin() as conn:
status.data_updated = await _get_database_date(conn)
status.database_version = await _get_database_version(conn)
except asyncpg.PostgresError:
return StatusResult(700, 'Database connection failed')
return status

42
nominatim/cli.py
View File

@@ -9,6 +9,7 @@ Command-line interface to the Nominatim functions for import, update,
database administration and querying.
"""
from typing import Optional, Any, List, Union
import importlib
import logging
import os
import sys
@@ -60,7 +61,7 @@ class CommandlineParser:
def nominatim_version_text(self) -> str:
""" Program name and version number as string
"""
text = f'Nominatim version {version.version_str()}'
text = f'Nominatim version {version.NOMINATIM_VERSION!s}'
if version.GIT_COMMIT_HASH is not None:
text += f' ({version.GIT_COMMIT_HASH})'
return text
@@ -197,10 +198,15 @@ class AdminServe:
"""\
Start a simple web server for serving the API.
This command starts the built-in PHP webserver to serve the website
This command starts a built-in webserver to serve the website
from the current project directory. This webserver is only suitable
for testing and development. Do not use it in production setups!
There are different webservers available. The default 'php' engine
runs the classic PHP frontend. The other engines are Python servers
which run the new Python frontend code. This is highly experimental
at the moment and may not include the full API.
By the default, the webserver can be accessed at: http://127.0.0.1:8088
"""
@@ -208,10 +214,40 @@ class AdminServe:
group = parser.add_argument_group('Server arguments')
group.add_argument('--server', default='127.0.0.1:8088',
help='The address the server will listen to.')
group.add_argument('--engine', default='php',
choices=('php', 'sanic', 'falcon', 'starlette'),
help='Webserver framework to run. (default: php)')
def run(self, args: NominatimArgs) -> int:
run_php_server(args.server, args.project_dir / 'website')
if args.engine == 'php':
run_php_server(args.server, args.project_dir / 'website')
else:
server_info = args.server.split(':', 1)
host = server_info[0]
if len(server_info) > 1:
if not server_info[1].isdigit():
raise UsageError('Invalid format for --server parameter. Use <host>:<port>')
port = int(server_info[1])
else:
port = 8088
if args.engine == 'sanic':
server_module = importlib.import_module('nominatim.server.sanic.server')
app = server_module.get_application(args.project_dir)
app.run(host=host, port=port, debug=True)
else:
import uvicorn # pylint: disable=import-outside-toplevel
if args.engine == 'falcon':
server_module = importlib.import_module('nominatim.server.falcon.server')
elif args.engine == 'starlette':
server_module = importlib.import_module('nominatim.server.starlette.server')
app = server_module.get_application(args.project_dir)
uvicorn.run(app, host=host, port=port)
return 0

12
nominatim/clicmd/api.py
View File

@@ -14,6 +14,9 @@ import logging
from nominatim.tools.exec_utils import run_api_script
from nominatim.errors import UsageError
from nominatim.clicmd.args import NominatimArgs
from nominatim.api import NominatimAPI
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
# Do not repeat documentation of subcommand classes.
# pylint: disable=C0111
@@ -264,7 +267,7 @@ class APIDetails:
class APIStatus:
"""\
"""
Execute API status query.
This command works exactly the same as if calling the /status endpoint on
@@ -274,10 +277,13 @@ class APIStatus:
"""
def add_args(self, parser: argparse.ArgumentParser) -> None:
formats = formatting.create(StatusResult).list_formats()
group = parser.add_argument_group('API parameters')
group.add_argument('--format', default='text', choices=['text', 'json'],
group.add_argument('--format', default=formats[0], choices=formats,
help='Format of result')
def run(self, args: NominatimArgs) -> int:
return _run_api('status', args, dict(format=args.format))
status = NominatimAPI(args.project_dir).status()
print(formatting.create(StatusResult).format(status, args.format))
return 0

1
nominatim/clicmd/args.py
View File

@@ -127,6 +127,7 @@ class NominatimArgs:
# Arguments to 'serve'
server: str
engine: str
# Arguments to 'special-phrases
import_from_wiki: bool

4
nominatim/clicmd/setup.py
View File

@@ -18,7 +18,7 @@ from nominatim.config import Configuration
from nominatim.db.connection import connect
from nominatim.db import status, properties
from nominatim.tokenizer.base import AbstractTokenizer
from nominatim.version import version_str
from nominatim.version import NOMINATIM_VERSION
from nominatim.clicmd.args import NominatimArgs
from nominatim.errors import UsageError
@@ -205,4 +205,4 @@ class SetupAll:
except Exception as exc: # pylint: disable=broad-except
LOG.error('Cannot determine date of database: %s', exc)
properties.set_property(conn, 'database_version', version_str())
properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))

15
nominatim/config.py
View File

@@ -17,6 +17,7 @@ import json
import yaml
from dotenv import dotenv_values
from psycopg2.extensions import parse_dsn
from nominatim.typing import StrPath
from nominatim.errors import UsageError
@@ -51,7 +52,7 @@ class Configuration:
Nominatim uses dotenv to configure the software. Configuration options
are resolved in the following order:
* from the OS environment (or the dirctionary given in `environ`
* from the OS environment (or the dictionary given in `environ`)
* from the .env file in the project directory of the installation
* from the default installation in the configuration directory
@@ -164,6 +165,18 @@ class Configuration:
return dsn
def get_database_params(self) -> Mapping[str, str]:
""" Get the configured parameters for the database connection
as a mapping.
"""
dsn = self.DATABASE_DSN
if dsn.startswith('pgsql:'):
return dict((p.split('=', 1) for p in dsn[6:].split(';')))
return parse_dsn(dsn)
def get_import_style_file(self) -> Path:
""" Return the import style file as a path object. Translates the
name of the standard styles automatically into a file in the

0
nominatim/result_formatter/__init__.py Normal file
View File

69
nominatim/result_formatter/base.py Normal file
View File

@@ -0,0 +1,69 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Helper classes and function for writing result formatting modules.
"""
from typing import Type, TypeVar, Dict, Mapping, List, Callable, Generic, Any
from collections import defaultdict
T = TypeVar('T') # pylint: disable=invalid-name
FormatFunc = Callable[[T], str]
class ResultFormatter(Generic[T]):
""" This class dispatches format calls to the appropriate formatting
function previously defined with the `format_func` decorator.
"""
def __init__(self, funcs: Mapping[str, FormatFunc[T]]) -> None:
self.functions = funcs
def list_formats(self) -> List[str]:
""" Return a list of formats supported by this formatter.
"""
return list(self.functions.keys())
def supports_format(self, fmt: str) -> bool:
""" Check if the given format is supported by this formatter.
"""
return fmt in self.functions
def format(self, result: T, fmt: str) -> str:
""" Convert the given result into a string using the given format.
The format is expected to be in the list returned by
`list_formats()`.
"""
return self.functions[fmt](result)
class FormatDispatcher:
""" A factory class for result formatters.
"""
def __init__(self) -> None:
self.format_functions: Dict[Type[Any], Dict[str, FormatFunc[Any]]] = defaultdict(dict)
def format_func(self, result_class: Type[T],
fmt: str) -> Callable[[FormatFunc[T]], FormatFunc[T]]:
""" Decorator for a function that formats a given type of result into the
selected format.
"""
def decorator(func: FormatFunc[T]) -> FormatFunc[T]:
self.format_functions[result_class][fmt] = func
return func
return decorator
def __call__(self, result_class: Type[T]) -> ResultFormatter[T]:
""" Create an instance of a format class for the given result type.
"""
return ResultFormatter(self.format_functions[result_class])

38
nominatim/result_formatter/v1.py Normal file
View File

@@ -0,0 +1,38 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Output formatters for API version v1.
"""
from typing import Dict, Any
from collections import OrderedDict
import json
from nominatim.result_formatter.base import FormatDispatcher
from nominatim.apicmd.status import StatusResult
create = FormatDispatcher()
@create.format_func(StatusResult, 'text')
def _format_status_text(result: StatusResult) -> str:
if result.status:
return f"ERROR: {result.message}"
return 'OK'
@create.format_func(StatusResult, 'json')
def _format_status_json(result: StatusResult) -> str:
out: Dict[str, Any] = OrderedDict()
out['status'] = result.status
out['message'] = result.message
if result.data_updated is not None:
out['data_updated'] = result.data_updated.isoformat()
out['software_version'] = str(result.software_version)
if result.database_version is not None:
out['database_version'] = str(result.database_version)
return json.dumps(out)

0
nominatim/server/__init__.py Normal file
View File

0
nominatim/server/falcon/__init__.py Normal file
View File

82
nominatim/server/falcon/server.py Normal file
View File

@@ -0,0 +1,82 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the falcon webserver framework.
"""
from typing import Type, Any, Optional, Mapping
from pathlib import Path
import falcon
import falcon.asgi
from nominatim.api import NominatimAPIAsync
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
CONTENT_TYPE = {
'text': falcon.MEDIA_TEXT,
'xml': falcon.MEDIA_XML
}
class NominatimV1:
""" Implementation of V1 version of the Nominatim API.
"""
def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]]) -> None:
self.api = NominatimAPIAsync(project_dir, environ)
self.formatters = {}
for rtype in (StatusResult, ):
self.formatters[rtype] = formatting.create(rtype)
def parse_format(self, req: falcon.asgi.Request, rtype: Type[Any], default: str) -> None:
""" Get and check the 'format' parameter and prepare the formatter.
`rtype` describes the expected return type and `default` the
format value to assume when no parameter is present.
"""
req.context.format = req.get_param('format', default=default)
req.context.formatter = self.formatters[rtype]
if not req.context.formatter.supports_format(req.context.format):
raise falcon.HTTPBadRequest(
description="Parameter 'format' must be one of: " +
', '.join(req.context.formatter.list_formats()))
def format_response(self, req: falcon.asgi.Request, resp: falcon.asgi.Response,
result: Any) -> None:
""" Render response into a string according to the formatter
set in `parse_format()`.
"""
resp.text = req.context.formatter.format(result, req.context.format)
resp.content_type = CONTENT_TYPE.get(req.context.format, falcon.MEDIA_JSON)
async def on_get_status(self, req: falcon.asgi.Request, resp: falcon.asgi.Response) -> None:
""" Implementation of status endpoint.
"""
self.parse_format(req, StatusResult, 'text')
result = await self.api.status()
self.format_response(req, resp, result)
if result.status and req.context.format == 'text':
resp.status = 500
def get_application(project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> falcon.asgi.App:
""" Create a Nominatim falcon ASGI application.
"""
app = falcon.asgi.App()
api = NominatimV1(project_dir, environ)
app.add_route('/status', api, suffix='status')
return app

0
nominatim/server/sanic/__init__.py Normal file
View File

86
nominatim/server/sanic/server.py Normal file
View File

@@ -0,0 +1,86 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the sanic webserver framework.
"""
from typing import Any, Optional, Mapping
from pathlib import Path
import sanic
from nominatim.api import NominatimAPIAsync
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
api = sanic.Blueprint('NominatimAPI')
CONTENT_TYPE = {
'text': 'text/plain; charset=utf-8',
'xml': 'text/xml; charset=utf-8'
}
def usage_error(msg: str) -> sanic.HTTPResponse:
""" Format the response for an error with the query parameters.
"""
return sanic.response.text(msg, status=400)
def api_response(request: sanic.Request, result: Any) -> sanic.HTTPResponse:
""" Render a response from the query results using the configured
formatter.
"""
body = request.ctx.formatter.format(result, request.ctx.format)
return sanic.response.text(body,
content_type=CONTENT_TYPE.get(request.ctx.format,
'application/json'))
@api.on_request # type: ignore[misc]
async def extract_format(request: sanic.Request) -> Optional[sanic.HTTPResponse]:
""" Get and check the 'format' parameter and prepare the formatter.
`ctx.result_type` describes the expected return type and
`ctx.default_format` the format value to assume when no parameter
is present.
"""
assert request.route is not None
request.ctx.formatter = request.app.ctx.formatters[request.route.ctx.result_type]
request.ctx.format = request.args.get('format', request.route.ctx.default_format)
if not request.ctx.formatter.supports_format(request.ctx.format):
return usage_error("Parameter 'format' must be one of: " +
', '.join(request.ctx.formatter.list_formats()))
return None
@api.get('/status', ctx_result_type=StatusResult, ctx_default_format='text')
async def status(request: sanic.Request) -> sanic.HTTPResponse:
""" Implementation of status endpoint.
"""
result = await request.app.ctx.api.status()
response = api_response(request, result)
if request.ctx.format == 'text' and result.status:
response.status = 500
return response
def get_application(project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> sanic.Sanic:
""" Create a Nominatim sanic ASGI application.
"""
app = sanic.Sanic("NominatimInstance")
app.ctx.api = NominatimAPIAsync(project_dir, environ)
app.ctx.formatters = {}
for rtype in (StatusResult, ):
app.ctx.formatters[rtype] = formatting.create(rtype)
app.blueprint(api)
return app

0
nominatim/server/starlette/__init__.py Normal file
View File

83
nominatim/server/starlette/server.py Normal file
View File

@@ -0,0 +1,83 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Server implementation using the starlette webserver framework.
"""
from typing import Any, Type, Optional, Mapping
from pathlib import Path
from starlette.applications import Starlette
from starlette.routing import Route
from starlette.exceptions import HTTPException
from starlette.responses import Response
from starlette.requests import Request
from nominatim.api import NominatimAPIAsync
from nominatim.apicmd.status import StatusResult
import nominatim.result_formatter.v1 as formatting
CONTENT_TYPE = {
'text': 'text/plain; charset=utf-8',
'xml': 'text/xml; charset=utf-8'
}
FORMATTERS = {
StatusResult: formatting.create(StatusResult)
}
def parse_format(request: Request, rtype: Type[Any], default: str) -> None:
""" Get and check the 'format' parameter and prepare the formatter.
`rtype` describes the expected return type and `default` the
format value to assume when no parameter is present.
"""
fmt = request.query_params.get('format', default=default)
fmtter = FORMATTERS[rtype]
if not fmtter.supports_format(fmt):
raise HTTPException(400, detail="Parameter 'format' must be one of: " +
', '.join(fmtter.list_formats()))
request.state.format = fmt
request.state.formatter = fmtter
def format_response(request: Request, result: Any) -> Response:
""" Render response into a string according to the formatter
set in `parse_format()`.
"""
fmt = request.state.format
return Response(request.state.formatter.format(result, fmt),
media_type=CONTENT_TYPE.get(fmt, 'application/json'))
async def on_status(request: Request) -> Response:
""" Implementation of status endpoint.
"""
parse_format(request, StatusResult, 'text')
result = await request.app.state.API.status()
response = format_response(request, result)
if request.state.format == 'text' and result.status:
response.status_code = 500
return response
V1_ROUTES = [
Route('/status', endpoint=on_status)
]
def get_application(project_dir: Path,
environ: Optional[Mapping[str, str]] = None) -> Starlette:
""" Create a Nominatim falcon ASGI application.
"""
app = Starlette(debug=True, routes=V1_ROUTES)
app.state.API = NominatimAPIAsync(project_dir, environ)
return app

6
nominatim/tools/collect_os_info.py
View File

@@ -20,7 +20,7 @@ from psycopg2.extensions import make_dsn, parse_dsn
from nominatim.config import Configuration
from nominatim.db.connection import connect
from nominatim.typing import DictCursorResults
from nominatim.version import version_str
from nominatim.version import NOMINATIM_VERSION
def convert_version(ver_tup: Tuple[int, int]) -> str:
@@ -135,8 +135,8 @@ def report_system_information(config: Configuration) -> None:
**Software Environment:**
- Python version: {sys.version}
- Nominatim version: {version_str()}
- PostgreSQL version: {postgresql_ver}
- Nominatim version: {NOMINATIM_VERSION!s}
- PostgreSQL version: {postgresql_ver}
- PostGIS version: {postgis_ver}
- OS: {os_name_info()}

4
nominatim/tools/exec_utils.py
View File

@@ -17,7 +17,7 @@ from urllib.parse import urlencode
from nominatim.config import Configuration
from nominatim.typing import StrPath
from nominatim.version import version_str
from nominatim.version import NOMINATIM_VERSION
from nominatim.db.connection import get_pg_env
LOG = logging.getLogger()
@@ -162,7 +162,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
def get_url(url: str) -> str:
""" Get the contents from the given URL and return it as a UTF-8 string.
"""
headers = {"User-Agent": f"Nominatim/{version_str()}"}
headers = {"User-Agent": f"Nominatim/{NOMINATIM_VERSION!s}"}
try:
request = urlrequest.Request(url, headers=headers)

21
nominatim/tools/migration.py
View File

@@ -15,16 +15,14 @@ from psycopg2 import sql as pysql
from nominatim.config import Configuration
from nominatim.db import properties
from nominatim.db.connection import connect, Connection
from nominatim.version import NOMINATIM_VERSION, version_str
from nominatim.version import NominatimVersion, NOMINATIM_VERSION, parse_version
from nominatim.tools import refresh
from nominatim.tokenizer import factory as tokenizer_factory
from nominatim.errors import UsageError
LOG = logging.getLogger()
VersionTuple = Tuple[int, int, int, int]
_MIGRATION_FUNCTIONS : List[Tuple[VersionTuple, Callable[..., None]]] = []
_MIGRATION_FUNCTIONS : List[Tuple[NominatimVersion, Callable[..., None]]] = []
def migrate(config: Configuration, paths: Any) -> int:
""" Check for the current database version and execute migrations,
@@ -37,8 +35,7 @@ def migrate(config: Configuration, paths: Any) -> int:
db_version_str = None
if db_version_str is not None:
parts = db_version_str.split('.')
db_version = tuple(int(x) for x in parts[:2] + parts[2].split('-'))
db_version = parse_version(db_version_str)
if db_version == NOMINATIM_VERSION:
LOG.warning("Database already at latest version (%s)", db_version_str)
@@ -53,8 +50,7 @@ def migrate(config: Configuration, paths: Any) -> int:
for version, func in _MIGRATION_FUNCTIONS:
if db_version <= version:
title = func.__doc__ or ''
LOG.warning("Running: %s (%s)", title.split('\n', 1)[0],
version_str(version))
LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version)
kwargs = dict(conn=conn, config=config, paths=paths)
func(**kwargs)
conn.commit()
@@ -66,14 +62,14 @@ def migrate(config: Configuration, paths: Any) -> int:
tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
tokenizer.update_sql_functions(config)
properties.set_property(conn, 'database_version', version_str())
properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))
conn.commit()
return 0
def _guess_version(conn: Connection) -> VersionTuple:
def _guess_version(conn: Connection) -> NominatimVersion:
""" Guess a database version when there is no property table yet.
Only migrations for 3.6 and later are supported, so bail out
when the version seems older.
@@ -89,7 +85,7 @@ def _guess_version(conn: Connection) -> VersionTuple:
'prior to 3.6.0. Automatic migration not possible.')
raise UsageError('Migration not possible.')
return (3, 5, 0, 99)
return NominatimVersion(3, 5, 0, 99)
@@ -108,7 +104,8 @@ def _migration(major: int, minor: int, patch: int = 0,
there.
"""
def decorator(func: Callable[..., None]) -> Callable[..., None]:
_MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
version = (NominatimVersion(major, minor, patch, dbpatch))
_MIGRATION_FUNCTIONS.append((version, func))
return func
return decorator

4
nominatim/tools/refresh.py
View File

@@ -18,7 +18,7 @@ from nominatim.config import Configuration
from nominatim.db.connection import Connection, connect
from nominatim.db.utils import execute_file
from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.version import version_str
from nominatim.version import NOMINATIM_VERSION
LOG = logging.getLogger()
@@ -223,7 +223,7 @@ def setup_website(basedir: Path, config: Configuration, conn: Connection) -> Non
@define('CONST_Debug', $_GET['debug'] ?? false);
@define('CONST_LibDir', '{config.lib_dir.php}');
@define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
@define('CONST_NominatimVersion', '{version_str()}');
@define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');
""")

57
nominatim/version.py
View File

@@ -7,25 +7,34 @@
"""
Version information for Nominatim.
"""
from typing import Optional, Tuple
from typing import Optional, NamedTuple
# Version information: major, minor, patch level, database patch level
#
# The first three numbers refer to the last released version.
#
# The database patch level tracks important changes between releases
# and must always be increased when there is a change to the database or code
# that requires a migration.
#
# When adding a migration on the development branch, raise the patch level
# to 99 to make sure that the migration is applied when updating from a
# patch release to the next minor version. Patch releases usually shouldn't
# have migrations in them. When they are needed, then make sure that the
# migration can be reapplied and set the migration version to the appropriate
# patch level when cherry-picking the commit with the migration.
#
# Released versions always have a database patch level of 0.
NOMINATIM_VERSION = (4, 2, 99, 0)
class NominatimVersion(NamedTuple):
""" Version information for Nominatim. We follow semantic versioning.
Major, minor and patch_level refer to the last released version.
The database patch level tracks important changes between releases
and must always be increased when there is a change to the database or code
that requires a migration.
When adding a migration on the development branch, raise the patch level
to 99 to make sure that the migration is applied when updating from a
patch release to the next minor version. Patch releases usually shouldn't
have migrations in them. When they are needed, then make sure that the
migration can be reapplied and set the migration version to the appropriate
patch level when cherry-picking the commit with the migration.
"""
major: int
minor: int
patch_level: int
db_patch_level: int
def __str__(self) -> str:
return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}"
NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 0)
POSTGRESQL_REQUIRED_VERSION = (9, 6)
POSTGIS_REQUIRED_VERSION = (2, 2)
@@ -37,9 +46,11 @@ POSTGIS_REQUIRED_VERSION = (2, 2)
GIT_COMMIT_HASH : Optional[str] = None
# pylint: disable=consider-using-f-string
def version_str(version:Tuple[int, int, int, int] = NOMINATIM_VERSION) -> str:
def parse_version(version: str) -> NominatimVersion:
""" Parse a version string into a version consisting of a tuple of
four ints: major, minor, patch level, database patch level
This is the reverse operation of `version_str()`.
"""
Return a human-readable string of the version.
"""
return '{}.{}.{}-{}'.format(*version)
parts = version.split('.')
return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')])

1
test/bdd/environment.py
View File

@@ -28,6 +28,7 @@ userconfig = {
'SERVER_MODULE_PATH' : None,
'TOKENIZER' : None, # Test with a custom tokenizer
'STYLE' : 'extratags',
'API_ENGINE': 'php',
'PHPCOV' : False, # set to output directory to enable code coverage
}

55
test/bdd/steps/nominatim_environment.py
View File

@@ -5,6 +5,7 @@
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
from pathlib import Path
import importlib
import sys
import tempfile
@@ -49,6 +50,12 @@ class NominatimEnvironment:
self.api_db_done = False
self.website_dir = None
self.api_engine = None
if config['API_ENGINE'] != 'php':
if not hasattr(self, f"create_api_request_func_{config['API_ENGINE']}"):
raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'")
self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")()
def connect_database(self, dbname):
""" Return a connection to the database with the given name.
Uses configured host, user and port.
@@ -323,3 +330,51 @@ class NominatimEnvironment:
WHERE class='place' and type='houses'
and osm_type='W'
and ST_GeometryType(geometry) = 'ST_LineString'""")
def create_api_request_func_starlette(self):
import nominatim.server.starlette.server
from asgi_lifespan import LifespanManager
import httpx
async def _request(endpoint, params, project_dir, environ):
app = nominatim.server.starlette.server.get_application(project_dir, environ)
async with LifespanManager(app):
async with httpx.AsyncClient(app=app, base_url="http://nominatim.test") as client:
response = await client.get(f"/{endpoint}", params=params)
return response.text, response.status_code
return _request
def create_api_request_func_sanic(self):
import nominatim.server.sanic.server
async def _request(endpoint, params, project_dir, environ):
app = nominatim.server.sanic.server.get_application(project_dir, environ)
_, response = await app.asgi_client.get(f"/{endpoint}", params=params)
return response.text, response.status_code
return _request
def create_api_request_func_falcon(self):
import nominatim.server.falcon.server
import falcon.testing
async def _request(endpoint, params, project_dir, environ):
app = nominatim.server.falcon.server.get_application(project_dir, environ)
async with falcon.testing.ASGIConductor(app) as conductor:
response = await conductor.get(f"/{endpoint}", params=params)
return response.text, response.status_code
return _request

12
test/bdd/steps/steps_api_queries.py
View File

@@ -9,10 +9,12 @@
Queries may either be run directly via PHP using the query script
or via the HTTP interface using php-cgi.
"""
from pathlib import Path
import json
import os
import re
import logging
import asyncio
from urllib.parse import urlencode
from utils import run_script
@@ -72,6 +74,16 @@ def send_api_query(endpoint, params, fmt, context):
for h in context.table.headings:
params[h] = context.table[0][h]
if context.nominatim.api_engine is None:
return send_api_query_php(endpoint, params, context)
return asyncio.run(context.nominatim.api_engine(endpoint, params,
Path(context.nominatim.website_dir.name),
context.nominatim.test_env))
def send_api_query_php(endpoint, params, context):
env = dict(BASE_SERVER_ENV)
env['QUERY_STRING'] = urlencode(params)

22
test/python/api/conftest.py Normal file
View File

@@ -0,0 +1,22 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Helper fixtures for API call tests.
"""
from pathlib import Path
import pytest
import time
from nominatim.api import NominatimAPI
@pytest.fixture
def apiobj(temp_db):
""" Create an asynchronous SQLAlchemy engine for the test DB.
"""
api = NominatimAPI(Path('/invalid'), {})
yield api
api.close()

60
test/python/api/test_api_status.py Normal file
View File

@@ -0,0 +1,60 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Tests for the status API call.
"""
from pathlib import Path
import datetime as dt
import pytest
from nominatim.version import NOMINATIM_VERSION, NominatimVersion
from nominatim.api import NominatimAPI
def test_status_no_extra_info(apiobj, table_factory):
table_factory('import_status',
definition="lastimportdate timestamp with time zone NOT NULL")
table_factory('nominatim_properties',
definition='property TEXT, value TEXT')
result = apiobj.status()
assert result.status == 0
assert result.message == 'OK'
assert result.software_version == NOMINATIM_VERSION
assert result.database_version is None
assert result.data_updated is None
def test_status_full(apiobj, table_factory):
table_factory('import_status',
definition="lastimportdate timestamp with time zone NOT NULL",
content=(('2022-12-07 15:14:46+01',),))
table_factory('nominatim_properties',
definition='property TEXT, value TEXT',
content=(('database_version', '99.5.4-2'), ))
result = apiobj.status()
assert result.status == 0
assert result.message == 'OK'
assert result.software_version == NOMINATIM_VERSION
assert result.database_version == NominatimVersion(99, 5, 4, 2)
assert result.data_updated == dt.datetime(2022, 12, 7, 14, 14, 46, 0, tzinfo=dt.timezone.utc)
def test_status_database_not_found(monkeypatch):
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'dbname=rgjdfkgjedkrgdfkngdfkg')
api = NominatimAPI(Path('/invalid'), {})
result = api.status()
assert result.status == 700
assert result.message == 'Database connection failed'
assert result.software_version == NOMINATIM_VERSION
assert result.database_version is None
assert result.data_updated is None

44
test/python/cli/test_cli.py
View File

@@ -11,6 +11,7 @@ These tests just check that the various command line parameters route to the
correct functionionality. They use a lot of monkeypatching to avoid executing
the actual functions.
"""
import importlib
import pytest
import nominatim.indexer.indexer
@@ -59,7 +60,7 @@ def test_cli_add_data_tiger_data(cli_call, cli_tokenizer_mock, mock_func_factory
assert mock.called == 1
def test_cli_serve_command(cli_call, mock_func_factory):
def test_cli_serve_php(cli_call, mock_func_factory):
func = mock_func_factory(nominatim.cli, 'run_php_server')
cli_call('serve') == 0
@@ -67,6 +68,47 @@ def test_cli_serve_command(cli_call, mock_func_factory):
assert func.called == 1
def test_cli_serve_sanic(cli_call, mock_func_factory):
mod = pytest.importorskip("sanic")
func = mock_func_factory(mod.Sanic, "run")
cli_call('serve', '--engine', 'sanic') == 0
assert func.called == 1
def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
pytest.importorskip("starlette")
mod = pytest.importorskip("uvicorn")
func = mock_func_factory(mod, "run")
cli_call('serve', '--engine', 'starlette', '--server', 'foobar:4545') == 0
assert func.called == 1
assert func.last_kwargs['host'] == 'foobar'
assert func.last_kwargs['port'] == 4545
def test_cli_serve_starlette_custom_server_bad_port(cli_call, mock_func_factory):
pytest.importorskip("starlette")
mod = pytest.importorskip("uvicorn")
func = mock_func_factory(mod, "run")
cli_call('serve', '--engine', 'starlette', '--server', 'foobar:45:45') == 1
@pytest.mark.parametrize("engine", ['falcon', 'starlette'])
def test_cli_serve_uvicorn_based(cli_call, engine, mock_func_factory):
pytest.importorskip(engine)
mod = pytest.importorskip("uvicorn")
func = mock_func_factory(mod, "run")
cli_call('serve', '--engine', engine) == 0
assert func.called == 1
assert func.last_kwargs['host'] == '127.0.0.1'
assert func.last_kwargs['port'] == 8088
def test_cli_export_command(cli_call, mock_run_legacy):
assert cli_call('export', '--output-all-postcodes') == 0

31
test/python/cli/test_cmd_api.py
View File

@@ -7,9 +7,12 @@
"""
Tests for API access commands of command-line interface wrapper.
"""
import json
import pytest
import nominatim.clicmd.api
import nominatim.api
from nominatim.apicmd.status import StatusResult
@pytest.mark.parametrize("endpoint", (('search', 'reverse', 'lookup', 'details', 'status')))
@@ -27,9 +30,8 @@ def test_no_api_without_phpcgi(endpoint):
('details', '--node', '1'),
('details', '--way', '1'),
('details', '--relation', '1'),
('details', '--place_id', '10001'),
('status',)])
class TestCliApiCall:
('details', '--place_id', '10001')])
class TestCliApiCallPhp:
@pytest.fixture(autouse=True)
def setup_cli_call(self, params, cli_call, mock_func_factory, tmp_path):
@@ -55,6 +57,29 @@ class TestCliApiCall:
assert self.run_nominatim() == 1
class TestCliStatusCall:
@pytest.fixture(autouse=True)
def setup_status_mock(self, monkeypatch):
monkeypatch.setattr(nominatim.api.NominatimAPI, 'status',
lambda self: StatusResult(200, 'OK'))
def test_status_simple(self, cli_call, tmp_path):
result = cli_call('status', '--project-dir', str(tmp_path))
assert result == 0
def test_status_json_format(self, cli_call, tmp_path, capsys):
result = cli_call('status', '--project-dir', str(tmp_path),
'--format', 'json')
assert result == 0
json.loads(capsys.readouterr().out)
QUERY_PARAMS = {
'search': ('--query', 'somewhere'),
'reverse': ('--lat', '20', '--lon', '30'),

63
test/python/result_formatter/test_v1.py Normal file
View File

@@ -0,0 +1,63 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Tests for formatting results for the V1 API.
"""
import datetime as dt
import pytest
import nominatim.result_formatter.v1 as format_module
from nominatim.apicmd.status import StatusResult
from nominatim.version import NOMINATIM_VERSION
STATUS_FORMATS = {'text', 'json'}
class TestStatusResultFormat:
@pytest.fixture(autouse=True)
def make_formatter(self):
self.formatter = format_module.create(StatusResult)
def test_format_list(self):
assert set(self.formatter.list_formats()) == STATUS_FORMATS
@pytest.mark.parametrize('fmt', list(STATUS_FORMATS))
def test_supported(self, fmt):
assert self.formatter.supports_format(fmt)
def test_unsupported(self):
assert not self.formatter.supports_format('gagaga')
def test_format_text(self):
assert self.formatter.format(StatusResult(0, 'message here'), 'text') == 'OK'
def test_format_text(self):
assert self.formatter.format(StatusResult(500, 'message here'), 'text') == 'ERROR: message here'
def test_format_json_minimal(self):
status = StatusResult(700, 'Bad format.')
result = self.formatter.format(status, 'json')
assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, )
def test_format_json_full(self):
status = StatusResult(0, 'OK')
status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc)
status.database_version = '5.6'
result = self.formatter.format(status, 'json')
assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, )

277
vagrant/Install-on-Ubuntu-18.sh
View File

@@ -1,277 +0,0 @@
#!/bin/bash -e
#
# hacks for broken vagrant box #DOCS:
sudo rm -f /var/lib/dpkg/lock #DOCS:
export APT_LISTCHANGES_FRONTEND=none #DOCS:
export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# *Note:* these installation instructions are also available in executable
# form for use with vagrant under vagrant/Install-on-Ubuntu-18.sh.
#
# Installing the Required Software
# ================================
#
# These instructions expect that you have a freshly installed Ubuntu 18.04.
#
# Make sure all packages are up-to-date by running:
#
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev\
libbz2-dev libpq-dev liblua5.3-dev lua5.3\
postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php-cli php-pgsql php-intl libicu-dev python3-pip \
python3-psutil python3-jinja2 python3-yaml python3-icu git
# Some of the Python packages that come with Ubuntu 18.04 are too old, so
# install the latest version from pip:
pip3 install --user python-dotenv datrie pyyaml psycopg2-binary
#
# System Configuration
# ====================
#
# The following steps are meant to configure a fresh Ubuntu installation
# for use with Nominatim. You may skip some of the steps if you have your
# OS already configured.
#
# Creating Dedicated User Accounts
# --------------------------------
#
# Nominatim will run as a global service on your machine. It is therefore
# best to install it under its own separate user account. In the following
# we assume this user is called nominatim and the installation will be in
# /srv/nominatim. To create the user and directory run:
#
# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
#
# You may find a more suitable location if you wish.
#
# To be able to copy and paste instructions from this manual, export
# user name and home directory now like this:
#
if [ "x$USERNAME" == "x" ]; then #DOCS:
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
fi #DOCS:
#
# **Never, ever run the installation as a root user.** You have been warned.
#
# Make sure that system servers can read from the home directory:
chmod a+x $USERHOME
# Setting up PostgreSQL
# ---------------------
#
# Tune the postgresql configuration, which is located in
# `/etc/postgresql/10/main/postgresql.conf`. See section *Postgres Tuning* in
# [the installation page](../admin/Installation.md#postgresql-tuning)
# for the parameters to change.
#
# Restart the postgresql service after updating this config file.
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo pg_ctlcluster 10 main start #DOCS:
else #DOCS:
sudo systemctl restart postgresql
fi #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
# the import and another for the webserver which should access the database
# for reading only:
#
sudo -u postgres createuser -s $USERNAME
sudo -u postgres createuser www-data
#
# Installing Nominatim
# ====================
#
# Building and Configuration
# --------------------------
#
# Get the source code from Github and change into the source directory
#
if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME
git clone --recursive https://github.com/openstreetmap/Nominatim.git
cd Nominatim
else #DOCS:
cd $USERHOME/Nominatim #DOCS:
fi #DOCS:
# When installing the latest source from github, you also need to
# download the country grid:
if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
fi #DOCS:
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
mkdir $USERHOME/build
cd $USERHOME/build
cmake $USERHOME/Nominatim
make
sudo make install
# Nominatim is now ready to use. You can continue with
# [importing a database from OSM data](../admin/Import.md). If you want to set up
# a webserver first, continue reading.
#
# Setting up a webserver
# ======================
#
# The webserver should serve the php scripts from the website directory of your
# [project directory](../admin/Import.md#creating-the-project-directory).
# This directory needs to exist when being configured.
# Therefore set up a project directory and create the website directory:
mkdir $USERHOME/nominatim-project
mkdir $USERHOME/nominatim-project/website
# The import process will populate the directory later.
#
# Option 1: Using Apache
# ----------------------
#
if [ "x$2" == "xinstall-apache" ]; then #DOCS:
#
# Apache has a PHP module that can be used to serve Nominatim. To install them
# run:
sudo apt install -y apache2 libapache2-mod-php
# You need to create an alias to the website directory in your apache
# configuration. Add a separate nominatim configuration to your webserver:
#DOCS:```sh
sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
<Directory "$USERHOME/nominatim-project/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim $USERHOME/nominatim-project/website
EOFAPACHECONF
#DOCS:```
#
# Then enable the configuration with
#
sudo a2enconf nominatim
# and restart apache:
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo apache2ctl start #DOCS:
else #DOCS:
sudo systemctl restart apache2
fi #DOCS:
# The Nominatim API is now available at `http://localhost/nominatim/`.
fi #DOCS:
#
# Option 2: Using nginx
# ---------------------
#
if [ "x$2" == "xinstall-nginx" ]; then #DOCS:
# Nginx has no native support for php scripts. You need to set up php-fpm for
# this purpose. First install nginx and php-fpm:
sudo apt install -y nginx php-fpm
# You need to configure php-fpm to listen on a Unix socket.
#DOCS:```sh
sudo tee /etc/php/7.2/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF
[www]
; Replace the tcp listener and add the unix socket
listen = /var/run/php-fpm-nominatim.sock
; Ensure that the daemon runs as the correct user
listen.owner = www-data
listen.group = www-data
listen.mode = 0666
; Unix user of FPM processes
user = www-data
group = www-data
; Choose process manager type (static, dynamic, ondemand)
pm = ondemand
pm.max_children = 5
EOF_PHP_FPM_CONF
#DOCS:```
# Then create a Nginx configuration to forward http requests to that socket.
#DOCS:```sh
sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
server {
listen 80 default_server;
listen [::]:80 default_server;
root $USERHOME/nominatim-project/website;
index search.php index.html;
location / {
try_files \$uri \$uri/ @php;
}
location @php {
fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php";
fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php";
fastcgi_param QUERY_STRING \$args;
fastcgi_pass unix:/var/run/php-fpm-nominatim.sock;
fastcgi_index index.php;
include fastcgi_params;
}
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f \$document_root\$fastcgi_script_name) {
return 404;
}
fastcgi_pass unix:/var/run/php-fpm-nominatim.sock;
fastcgi_index search.php;
include fastcgi.conf;
}
}
EOF_NGINX_CONF
#DOCS:```
#
# Enable the configuration and restart Nginx
#
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS:
sudo /usr/sbin/nginx & #DOCS:
else #DOCS:
sudo systemctl restart php7.2-fpm nginx
fi #DOCS:
# The Nominatim API is now available at `http://localhost/`.
fi #DOCS:

8
vagrant/Install-on-Ubuntu-20.sh
View File

@@ -27,9 +27,15 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
python3-psycopg2 python3-psutil python3-jinja2 \
python3-psycopg2 python3-psutil python3-jinja2 python3-pip \
python3-icu python3-datrie python3-yaml git
# Some of the Python packages that come with Ubuntu 20.04 are too old, so
# install the latest version from pip:
pip3 install --user sqlalchemy asyncpg
#
# System Configuration
# ====================

3
vagrant/Install-on-Ubuntu-22.sh
View File

@@ -28,7 +28,8 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-contrib-14 postgresql-14-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
python3-psycopg2 python3-psutil python3-jinja2 \
python3-icu python3-datrie git
python3-icu python3-datrie python3-sqlalchemy \
python3-asyncpg git
#
# System Configuration