Merge pull request #3121 from lonvia/port-remaining-api-calls

Port remaining API endpoints to Python

nominatim/api/v1/format.py

@@ -7,8 +7,9 @@
 """
 Output formatters for API version v1.
 """
-from typing import Mapping, Any
+from typing import List, Dict, Mapping, Any
 import collections
+import datetime as dt
 
 import nominatim.api as napi
 from nominatim.api.result_formatting import FormatDispatcher
@@ -16,6 +17,10 @@ from nominatim.api.v1.classtypes import ICONS
 from nominatim.api.v1 import format_json, format_xml
 from nominatim.utils.json_writer import JsonWriter
 
+class RawDataList(List[Dict[str, Any]]):
+    """ Data type for formatting raw data lists 'as is' in json.
+    """
+
 dispatch = FormatDispatcher()
 
 @dispatch.format_func(napi.StatusResult, 'text')
@@ -232,3 +237,20 @@ def _format_search_jsonv2(results: napi.SearchResults,
                            options: Mapping[str, Any]) -> str:
     return format_json.format_base_json(results, options, False,
                                         class_label='category')
+
+@dispatch.format_func(RawDataList, 'json')
+def _format_raw_data_json(results: RawDataList,  _: Mapping[str, Any]) -> str:
+    out = JsonWriter()
+    out.start_array()
+    for res in results:
+        out.start_object()
+        for k, v in res.items():
+            if isinstance(v, dt.datetime):
+                out.keyval(k, v.isoformat(sep= ' ', timespec='seconds'))
+            else:
+                out.keyval(k, v)
+        out.end_object().next()
+
+    out.end_array()
+
+    return out()

nominatim/api/v1/server_glue.py

@@ -15,11 +15,14 @@ import dataclasses
 import math
 from urllib.parse import urlencode
 
+import sqlalchemy as sa
+
 from nominatim.errors import UsageError
 from nominatim.config import Configuration
 import nominatim.api as napi
 import nominatim.api.logging as loglib
 from nominatim.api.v1.format import dispatch as formatting
+from nominatim.api.v1.format import RawDataList
 from nominatim.api.v1 import helpers
 
 CONTENT_TYPE = {
@@ -494,6 +497,58 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A
     return params.build_response(output)
 
 
+async def deletable_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any:
+    """ Server glue for /deletable endpoint.
+        This is a special endpoint that shows polygons that have been
+        deleted or are broken in the OSM data but are kept in the
+        Nominatim database to minimize disruption.
+    """
+    fmt = params.parse_format(RawDataList, 'json')
+
+    async with api.begin() as conn:
+        sql = sa.text(""" SELECT p.place_id, country_code,
+                                 name->'name' as name, i.*
+                          FROM placex p, import_polygon_delete i
+                          WHERE p.osm_id = i.osm_id AND p.osm_type = i.osm_type
+                                AND p.class = i.class AND p.type = i.type
+                      """)
+        results = RawDataList(r._asdict() for r in await conn.execute(sql))
+
+    return params.build_response(formatting.format_result(results, fmt, {}))
+
+
+async def polygons_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any:
+    """ Server glue for /polygons endpoint.
+        This is a special endpoint that shows polygons that have changed
+        thier size but are kept in the Nominatim database with their
+        old area to minimize disruption.
+    """
+    fmt = params.parse_format(RawDataList, 'json')
+    sql_params: Dict[str, Any] = {
+        'days': params.get_int('days', -1),
+        'cls': params.get('class')
+    }
+    reduced = params.get_bool('reduced', False)
+
+    async with api.begin() as conn:
+        sql = sa.select(sa.text("""osm_type, osm_id, class, type,
+                                   name->'name' as name,
+                                   country_code, errormessage, updated"""))\
+                .select_from(sa.text('import_polygon_error'))
+        if sql_params['days'] > 0:
+            sql = sql.where(sa.text("updated > 'now'::timestamp - make_interval(days => :days)"))
+        if reduced:
+            sql = sql.where(sa.text("errormessage like 'Area reduced%'"))
+        if sql_params['cls'] is not None:
+            sql = sql.where(sa.text("class = :cls"))
+
+        sql = sql.order_by(sa.literal_column('updated').desc()).limit(1000)
+
+        results = RawDataList(r._asdict() for r in await conn.execute(sql, sql_params))
+
+    return params.build_response(formatting.format_result(results, fmt, {}))
+
+
 EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any]
 
 ROUTES = [
@@ -501,5 +556,7 @@ ROUTES = [
     ('details', details_endpoint),
     ('reverse', reverse_endpoint),
     ('lookup', lookup_endpoint),
-    ('search', search_endpoint)
+    ('search', search_endpoint),
+    ('deletable', deletable_endpoint),
+    ('polygons', polygons_endpoint),
 ]

test/python/api/fake_adaptor.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Provides dummy implementations of ASGIAdaptor for testing.
+"""
+from collections import namedtuple
+
+import nominatim.api.v1.server_glue as glue
+from nominatim.config import Configuration
+
+class FakeError(BaseException):
+
+    def __init__(self, msg, status):
+        self.msg = msg
+        self.status = status
+
+    def __str__(self):
+        return f'{self.status} -- {self.msg}'
+
+FakeResponse = namedtuple('FakeResponse', ['status', 'output', 'content_type'])
+
+class FakeAdaptor(glue.ASGIAdaptor):
+
+    def __init__(self, params=None, headers=None, config=None):
+        self.params = params or {}
+        self.headers = headers or {}
+        self._config = config or Configuration(None)
+
+
+    def get(self, name, default=None):
+        return self.params.get(name, default)
+
+
+    def get_header(self, name, default=None):
+        return self.headers.get(name, default)
+
+
+    def error(self, msg, status=400):
+        return FakeError(msg, status)
+
+
+    def create_response(self, status, output):
+        return FakeResponse(status, output, self.content_type)
+
+
+    def config(self):
+        return self._config
+

test/python/api/test_api_deletable_v1.py

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Tests for the deletable v1 API call.
+"""
+import json
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+
+import psycopg2.extras
+
+from fake_adaptor import FakeAdaptor, FakeError, FakeResponse
+
+import nominatim.api.v1.server_glue as glue
+import nominatim.api as napi
+
+@pytest_asyncio.fixture
+async def api():
+    api = napi.NominatimAPIAsync(Path('/invalid'))
+    yield api
+    await api.close()
+
+
+class TestDeletableEndPoint:
+
+    @pytest.fixture(autouse=True)
+    def setup_deletable_table(self, temp_db_cursor, table_factory, temp_db_with_extensions):
+        psycopg2.extras.register_hstore(temp_db_cursor)
+        table_factory('import_polygon_delete',
+                      definition='osm_id bigint, osm_type char(1), class text, type text',
+                      content=[(345, 'N', 'boundary', 'administrative'),
+                               (781, 'R', 'landuse', 'wood'),
+                               (781, 'R', 'landcover', 'grass')])
+        table_factory('placex',
+                      definition="""place_id bigint, osm_id bigint, osm_type char(1),
+                                    class text, type text, name HSTORE, country_code char(2)""",
+                      content=[(1, 345, 'N', 'boundary', 'administrative', {'old_name': 'Former'}, 'ab'),
+                               (2, 781, 'R', 'landuse', 'wood', {'name': 'Wood'}, 'cd'),
+                               (3, 781, 'R', 'landcover', 'grass', None, 'cd')])
+
+
+
+    @pytest.mark.asyncio
+    async def test_deletable(self, api):
+        a = FakeAdaptor()
+
+        resp = await glue.deletable_endpoint(api, a)
+        results = json.loads(resp.output)
+
+        results.sort(key=lambda r: r['place_id'])
+
+        assert results == [{'place_id': 1, 'country_code': 'ab', 'name': None,
+                            'osm_id': 345, 'osm_type': 'N',
+                            'class': 'boundary', 'type': 'administrative'},
+                           {'place_id': 2, 'country_code': 'cd', 'name': 'Wood',
+                            'osm_id': 781, 'osm_type': 'R',
+                            'class': 'landuse', 'type': 'wood'},
+                           {'place_id': 3, 'country_code': 'cd', 'name': None,
+                            'osm_id': 781, 'osm_type': 'R',
+                            'class': 'landcover', 'type': 'grass'}]
+

test/python/api/test_api_polygons_v1.py

@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Tests for the deletable v1 API call.
+"""
+import json
+import datetime as dt
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+
+import psycopg2.extras
+
+from fake_adaptor import FakeAdaptor, FakeError, FakeResponse
+
+import nominatim.api.v1.server_glue as glue
+import nominatim.api as napi
+
+@pytest_asyncio.fixture
+async def api():
+    api = napi.NominatimAPIAsync(Path('/invalid'))
+    yield api
+    await api.close()
+
+
+class TestPolygonsEndPoint:
+
+    @pytest.fixture(autouse=True)
+    def setup_deletable_table(self, temp_db_cursor, table_factory, temp_db_with_extensions):
+        psycopg2.extras.register_hstore(temp_db_cursor)
+
+        self.now = dt.datetime.now()
+        self.recent = dt.datetime.now() - dt.timedelta(days=3)
+
+        table_factory('import_polygon_error',
+                      definition="""osm_id bigint,
+                                    osm_type character(1),
+                                    class text,
+                                    type text,
+                                    name hstore,
+                                    country_code character varying(2),
+                                    updated timestamp without time zone,
+                                    errormessage text,
+                                    prevgeometry geometry(Geometry,4326),
+                                    newgeometry geometry(Geometry,4326)""",
+                    content=[(345, 'N', 'boundary', 'administrative',
+                              {'name': 'Foo'}, 'xx', self.recent,
+                              'some text', None, None),
+                             (781, 'R', 'landuse', 'wood',
+                              None, 'ds', self.now,
+                              'Area reduced by lots', None, None)])
+
+
+    @pytest.mark.asyncio
+    async def test_polygons_simple(self, api):
+        a = FakeAdaptor()
+
+        resp = await glue.polygons_endpoint(api, a)
+        results = json.loads(resp.output)
+
+        results.sort(key=lambda r: (r['osm_type'], r['osm_id']))
+
+        assert results == [{'osm_type': 'N', 'osm_id': 345,
+                            'class': 'boundary', 'type': 'administrative',
+                            'name': 'Foo', 'country_code': 'xx',
+                            'errormessage': 'some text',
+                            'updated': self.recent.isoformat(sep=' ', timespec='seconds')},
+                           {'osm_type': 'R', 'osm_id': 781,
+                            'class': 'landuse', 'type': 'wood',
+                            'name': None, 'country_code': 'ds',
+                            'errormessage': 'Area reduced by lots',
+                            'updated': self.now.isoformat(sep=' ', timespec='seconds')}]
+
+
+    @pytest.mark.asyncio
+    async def test_polygons_days(self, api):
+        a = FakeAdaptor()
+        a.params['days'] = '2'
+
+        resp = await glue.polygons_endpoint(api, a)
+        results = json.loads(resp.output)
+
+        assert [r['osm_id'] for r in results] == [781]
+
+
+    @pytest.mark.asyncio
+    async def test_polygons_class(self, api):
+        a = FakeAdaptor()
+        a.params['class'] = 'landuse'
+
+        resp = await glue.polygons_endpoint(api, a)
+        results = json.loads(resp.output)
+
+        assert [r['osm_id'] for r in results] == [781]
+
+
+
+    @pytest.mark.asyncio
+    async def test_polygons_reduced(self, api):
+        a = FakeAdaptor()
+        a.params['reduced'] = '1'
+
+        resp = await glue.polygons_endpoint(api, a)
+        results = json.loads(resp.output)
+
+        assert [r['osm_id'] for r in results] == [781]

test/python/api/test_server_glue_v1.py

@@ -7,56 +7,18 @@
 """
 Tests for the Python web frameworks adaptor, v1 API.
 """
-from collections import namedtuple
 import json
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
 import pytest
 
-from nominatim.config import Configuration
+from fake_adaptor import FakeAdaptor, FakeError, FakeResponse
+
 import nominatim.api.v1.server_glue as glue
 import nominatim.api as napi
 import nominatim.api.logging as loglib
 
-class FakeError(BaseException):
-
-    def __init__(self, msg, status):
-        self.msg = msg
-        self.status = status
-
-    def __str__(self):
-        return f'{self.status} -- {self.msg}'
-
-FakeResponse = namedtuple('FakeResponse', ['status', 'output', 'content_type'])
-
-class FakeAdaptor(glue.ASGIAdaptor):
-
-    def __init__(self, params=None, headers=None, config=None):
-        self.params = params or {}
-        self.headers = headers or {}
-        self._config = config or Configuration(None)
-
-
-    def get(self, name, default=None):
-        return self.params.get(name, default)
-
-
-    def get_header(self, name, default=None):
-        return self.headers.get(name, default)
-
-
-    def error(self, msg, status=400):
-        return FakeError(msg, status)
-
-
-    def create_response(self, status, output):
-        return FakeResponse(status, output, self.content_type)
-
-
-    def config(self):
-        return self._config
-
 
 # ASGIAdaptor.get_int/bool()