mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-02-14 10:27:57 +00:00
Merge pull request #3121 from lonvia/port-remaining-api-calls
Port remaining API endpoints to Python
This commit is contained in:
Sarah Hoffmann
@@ -7,8 +7,9 @@
|
||||
"""
|
||||
Output formatters for API version v1.
|
||||
"""
|
||||
from typing import Mapping, Any
|
||||
from typing import List, Dict, Mapping, Any
|
||||
import collections
|
||||
import datetime as dt
|
||||
|
||||
import nominatim.api as napi
|
||||
from nominatim.api.result_formatting import FormatDispatcher
|
||||
@@ -16,6 +17,10 @@ from nominatim.api.v1.classtypes import ICONS
|
||||
from nominatim.api.v1 import format_json, format_xml
|
||||
from nominatim.utils.json_writer import JsonWriter
|
||||
|
||||
class RawDataList(List[Dict[str, Any]]):
|
||||
""" Data type for formatting raw data lists 'as is' in json.
|
||||
"""
|
||||
|
||||
dispatch = FormatDispatcher()
|
||||
|
||||
@dispatch.format_func(napi.StatusResult, 'text')
|
||||
@@ -232,3 +237,20 @@ def _format_search_jsonv2(results: napi.SearchResults,
|
||||
options: Mapping[str, Any]) -> str:
|
||||
return format_json.format_base_json(results, options, False,
|
||||
class_label='category')
|
||||
|
||||
@dispatch.format_func(RawDataList, 'json')
|
||||
def _format_raw_data_json(results: RawDataList, _: Mapping[str, Any]) -> str:
|
||||
out = JsonWriter()
|
||||
out.start_array()
|
||||
for res in results:
|
||||
out.start_object()
|
||||
for k, v in res.items():
|
||||
if isinstance(v, dt.datetime):
|
||||
out.keyval(k, v.isoformat(sep= ' ', timespec='seconds'))
|
||||
else:
|
||||
out.keyval(k, v)
|
||||
out.end_object().next()
|
||||
|
||||
out.end_array()
|
||||
|
||||
return out()
|
||||
|
||||
@@ -15,11 +15,14 @@ import dataclasses
|
||||
import math
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from nominatim.errors import UsageError
|
||||
from nominatim.config import Configuration
|
||||
import nominatim.api as napi
|
||||
import nominatim.api.logging as loglib
|
||||
from nominatim.api.v1.format import dispatch as formatting
|
||||
from nominatim.api.v1.format import RawDataList
|
||||
from nominatim.api.v1 import helpers
|
||||
|
||||
CONTENT_TYPE = {
|
||||
@@ -494,6 +497,58 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A
|
||||
return params.build_response(output)
|
||||
|
||||
|
||||
async def deletable_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
""" Server glue for /deletable endpoint.
|
||||
This is a special endpoint that shows polygons that have been
|
||||
deleted or are broken in the OSM data but are kept in the
|
||||
Nominatim database to minimize disruption.
|
||||
"""
|
||||
fmt = params.parse_format(RawDataList, 'json')
|
||||
|
||||
async with api.begin() as conn:
|
||||
sql = sa.text(""" SELECT p.place_id, country_code,
|
||||
name->'name' as name, i.*
|
||||
FROM placex p, import_polygon_delete i
|
||||
WHERE p.osm_id = i.osm_id AND p.osm_type = i.osm_type
|
||||
AND p.class = i.class AND p.type = i.type
|
||||
""")
|
||||
results = RawDataList(r._asdict() for r in await conn.execute(sql))
|
||||
|
||||
return params.build_response(formatting.format_result(results, fmt, {}))
|
||||
|
||||
|
||||
async def polygons_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
""" Server glue for /polygons endpoint.
|
||||
This is a special endpoint that shows polygons that have changed
|
||||
thier size but are kept in the Nominatim database with their
|
||||
old area to minimize disruption.
|
||||
"""
|
||||
fmt = params.parse_format(RawDataList, 'json')
|
||||
sql_params: Dict[str, Any] = {
|
||||
'days': params.get_int('days', -1),
|
||||
'cls': params.get('class')
|
||||
}
|
||||
reduced = params.get_bool('reduced', False)
|
||||
|
||||
async with api.begin() as conn:
|
||||
sql = sa.select(sa.text("""osm_type, osm_id, class, type,
|
||||
name->'name' as name,
|
||||
country_code, errormessage, updated"""))\
|
||||
.select_from(sa.text('import_polygon_error'))
|
||||
if sql_params['days'] > 0:
|
||||
sql = sql.where(sa.text("updated > 'now'::timestamp - make_interval(days => :days)"))
|
||||
if reduced:
|
||||
sql = sql.where(sa.text("errormessage like 'Area reduced%'"))
|
||||
if sql_params['cls'] is not None:
|
||||
sql = sql.where(sa.text("class = :cls"))
|
||||
|
||||
sql = sql.order_by(sa.literal_column('updated').desc()).limit(1000)
|
||||
|
||||
results = RawDataList(r._asdict() for r in await conn.execute(sql, sql_params))
|
||||
|
||||
return params.build_response(formatting.format_result(results, fmt, {}))
|
||||
|
||||
|
||||
EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any]
|
||||
|
||||
ROUTES = [
|
||||
@@ -501,5 +556,7 @@ ROUTES = [
|
||||
('details', details_endpoint),
|
||||
('reverse', reverse_endpoint),
|
||||
('lookup', lookup_endpoint),
|
||||
('search', search_endpoint)
|
||||
('search', search_endpoint),
|
||||
('deletable', deletable_endpoint),
|
||||
('polygons', polygons_endpoint),
|
||||
]
|
||||
|
||||
52
test/python/api/fake_adaptor.py
Normal file
52
test/python/api/fake_adaptor.py
Normal file
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2023 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Provides dummy implementations of ASGIAdaptor for testing.
|
||||
"""
|
||||
from collections import namedtuple
|
||||
|
||||
import nominatim.api.v1.server_glue as glue
|
||||
from nominatim.config import Configuration
|
||||
|
||||
class FakeError(BaseException):
|
||||
|
||||
def __init__(self, msg, status):
|
||||
self.msg = msg
|
||||
self.status = status
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.status} -- {self.msg}'
|
||||
|
||||
FakeResponse = namedtuple('FakeResponse', ['status', 'output', 'content_type'])
|
||||
|
||||
class FakeAdaptor(glue.ASGIAdaptor):
|
||||
|
||||
def __init__(self, params=None, headers=None, config=None):
|
||||
self.params = params or {}
|
||||
self.headers = headers or {}
|
||||
self._config = config or Configuration(None)
|
||||
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self.params.get(name, default)
|
||||
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return self.headers.get(name, default)
|
||||
|
||||
|
||||
def error(self, msg, status=400):
|
||||
return FakeError(msg, status)
|
||||
|
||||
|
||||
def create_response(self, status, output):
|
||||
return FakeResponse(status, output, self.content_type)
|
||||
|
||||
|
||||
def config(self):
|
||||
return self._config
|
||||
|
||||
67
test/python/api/test_api_deletable_v1.py
Normal file
67
test/python/api/test_api_deletable_v1.py
Normal file
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2023 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Tests for the deletable v1 API call.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
import psycopg2.extras
|
||||
|
||||
from fake_adaptor import FakeAdaptor, FakeError, FakeResponse
|
||||
|
||||
import nominatim.api.v1.server_glue as glue
|
||||
import nominatim.api as napi
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def api():
|
||||
api = napi.NominatimAPIAsync(Path('/invalid'))
|
||||
yield api
|
||||
await api.close()
|
||||
|
||||
|
||||
class TestDeletableEndPoint:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_deletable_table(self, temp_db_cursor, table_factory, temp_db_with_extensions):
|
||||
psycopg2.extras.register_hstore(temp_db_cursor)
|
||||
table_factory('import_polygon_delete',
|
||||
definition='osm_id bigint, osm_type char(1), class text, type text',
|
||||
content=[(345, 'N', 'boundary', 'administrative'),
|
||||
(781, 'R', 'landuse', 'wood'),
|
||||
(781, 'R', 'landcover', 'grass')])
|
||||
table_factory('placex',
|
||||
definition="""place_id bigint, osm_id bigint, osm_type char(1),
|
||||
class text, type text, name HSTORE, country_code char(2)""",
|
||||
content=[(1, 345, 'N', 'boundary', 'administrative', {'old_name': 'Former'}, 'ab'),
|
||||
(2, 781, 'R', 'landuse', 'wood', {'name': 'Wood'}, 'cd'),
|
||||
(3, 781, 'R', 'landcover', 'grass', None, 'cd')])
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deletable(self, api):
|
||||
a = FakeAdaptor()
|
||||
|
||||
resp = await glue.deletable_endpoint(api, a)
|
||||
results = json.loads(resp.output)
|
||||
|
||||
results.sort(key=lambda r: r['place_id'])
|
||||
|
||||
assert results == [{'place_id': 1, 'country_code': 'ab', 'name': None,
|
||||
'osm_id': 345, 'osm_type': 'N',
|
||||
'class': 'boundary', 'type': 'administrative'},
|
||||
{'place_id': 2, 'country_code': 'cd', 'name': 'Wood',
|
||||
'osm_id': 781, 'osm_type': 'R',
|
||||
'class': 'landuse', 'type': 'wood'},
|
||||
{'place_id': 3, 'country_code': 'cd', 'name': None,
|
||||
'osm_id': 781, 'osm_type': 'R',
|
||||
'class': 'landcover', 'type': 'grass'}]
|
||||
|
||||
111
test/python/api/test_api_polygons_v1.py
Normal file
111
test/python/api/test_api_polygons_v1.py
Normal file
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2023 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Tests for the deletable v1 API call.
|
||||
"""
|
||||
import json
|
||||
import datetime as dt
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
import psycopg2.extras
|
||||
|
||||
from fake_adaptor import FakeAdaptor, FakeError, FakeResponse
|
||||
|
||||
import nominatim.api.v1.server_glue as glue
|
||||
import nominatim.api as napi
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def api():
|
||||
api = napi.NominatimAPIAsync(Path('/invalid'))
|
||||
yield api
|
||||
await api.close()
|
||||
|
||||
|
||||
class TestPolygonsEndPoint:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_deletable_table(self, temp_db_cursor, table_factory, temp_db_with_extensions):
|
||||
psycopg2.extras.register_hstore(temp_db_cursor)
|
||||
|
||||
self.now = dt.datetime.now()
|
||||
self.recent = dt.datetime.now() - dt.timedelta(days=3)
|
||||
|
||||
table_factory('import_polygon_error',
|
||||
definition="""osm_id bigint,
|
||||
osm_type character(1),
|
||||
class text,
|
||||
type text,
|
||||
name hstore,
|
||||
country_code character varying(2),
|
||||
updated timestamp without time zone,
|
||||
errormessage text,
|
||||
prevgeometry geometry(Geometry,4326),
|
||||
newgeometry geometry(Geometry,4326)""",
|
||||
content=[(345, 'N', 'boundary', 'administrative',
|
||||
{'name': 'Foo'}, 'xx', self.recent,
|
||||
'some text', None, None),
|
||||
(781, 'R', 'landuse', 'wood',
|
||||
None, 'ds', self.now,
|
||||
'Area reduced by lots', None, None)])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polygons_simple(self, api):
|
||||
a = FakeAdaptor()
|
||||
|
||||
resp = await glue.polygons_endpoint(api, a)
|
||||
results = json.loads(resp.output)
|
||||
|
||||
results.sort(key=lambda r: (r['osm_type'], r['osm_id']))
|
||||
|
||||
assert results == [{'osm_type': 'N', 'osm_id': 345,
|
||||
'class': 'boundary', 'type': 'administrative',
|
||||
'name': 'Foo', 'country_code': 'xx',
|
||||
'errormessage': 'some text',
|
||||
'updated': self.recent.isoformat(sep=' ', timespec='seconds')},
|
||||
{'osm_type': 'R', 'osm_id': 781,
|
||||
'class': 'landuse', 'type': 'wood',
|
||||
'name': None, 'country_code': 'ds',
|
||||
'errormessage': 'Area reduced by lots',
|
||||
'updated': self.now.isoformat(sep=' ', timespec='seconds')}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polygons_days(self, api):
|
||||
a = FakeAdaptor()
|
||||
a.params['days'] = '2'
|
||||
|
||||
resp = await glue.polygons_endpoint(api, a)
|
||||
results = json.loads(resp.output)
|
||||
|
||||
assert [r['osm_id'] for r in results] == [781]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polygons_class(self, api):
|
||||
a = FakeAdaptor()
|
||||
a.params['class'] = 'landuse'
|
||||
|
||||
resp = await glue.polygons_endpoint(api, a)
|
||||
results = json.loads(resp.output)
|
||||
|
||||
assert [r['osm_id'] for r in results] == [781]
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_polygons_reduced(self, api):
|
||||
a = FakeAdaptor()
|
||||
a.params['reduced'] = '1'
|
||||
|
||||
resp = await glue.polygons_endpoint(api, a)
|
||||
results = json.loads(resp.output)
|
||||
|
||||
assert [r['osm_id'] for r in results] == [781]
|
||||
@@ -7,56 +7,18 @@
|
||||
"""
|
||||
Tests for the Python web frameworks adaptor, v1 API.
|
||||
"""
|
||||
from collections import namedtuple
|
||||
import json
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nominatim.config import Configuration
|
||||
from fake_adaptor import FakeAdaptor, FakeError, FakeResponse
|
||||
|
||||
import nominatim.api.v1.server_glue as glue
|
||||
import nominatim.api as napi
|
||||
import nominatim.api.logging as loglib
|
||||
|
||||
class FakeError(BaseException):
|
||||
|
||||
def __init__(self, msg, status):
|
||||
self.msg = msg
|
||||
self.status = status
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.status} -- {self.msg}'
|
||||
|
||||
FakeResponse = namedtuple('FakeResponse', ['status', 'output', 'content_type'])
|
||||
|
||||
class FakeAdaptor(glue.ASGIAdaptor):
|
||||
|
||||
def __init__(self, params=None, headers=None, config=None):
|
||||
self.params = params or {}
|
||||
self.headers = headers or {}
|
||||
self._config = config or Configuration(None)
|
||||
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self.params.get(name, default)
|
||||
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return self.headers.get(name, default)
|
||||
|
||||
|
||||
def error(self, msg, status=400):
|
||||
return FakeError(msg, status)
|
||||
|
||||
|
||||
def create_response(self, status, output):
|
||||
return FakeResponse(status, output, self.content_type)
|
||||
|
||||
|
||||
def config(self):
|
||||
return self._config
|
||||
|
||||
|
||||
# ASGIAdaptor.get_int/bool()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user