switch details cli command to new Python implementation

nominatim/api/__init__.py

@@ -28,3 +28,4 @@ from .results import (SourceTable as SourceTable,
                       WordInfo as WordInfo,
                       WordInfos as WordInfos,
                       SearchResult as SearchResult)
+from .localization import (Locales as Locales)

nominatim/api/localization.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Helper functions for localizing names of results.
+"""
+from typing import Mapping, List, Optional
+
+import re
+
+class Locales:
+    """ Helper class for localization of names.
+
+        It takes a list of language prefixes in their order of preferred
+        usage.
+    """
+
+    def __init__(self, langs: Optional[List[str]] = None):
+        self.languages = langs or []
+        self.name_tags: List[str] = []
+
+        # Build the list of supported tags. It is currently hard-coded.
+        self._add_lang_tags('name')
+        self._add_tags('name', 'brand')
+        self._add_lang_tags('official_name', 'short_name')
+        self._add_tags('official_name', 'short_name', 'ref')
+
+
+    def __bool__(self) -> bool:
+        return len(self.languages) > 0
+
+
+    def _add_tags(self, *tags: str) -> None:
+        for tag in tags:
+            self.name_tags.append(tag)
+            self.name_tags.append(f"_place_{tag}")
+
+
+    def _add_lang_tags(self, *tags: str) -> None:
+        for tag in tags:
+            for lang in self.languages:
+                self.name_tags.append(f"{tag}:{lang}")
+                self.name_tags.append(f"_place_{tag}:{lang}")
+
+
+    def display_name(self, names: Optional[Mapping[str, str]]) -> str:
+        """ Return the best matching name from a dictionary of names
+            containing different name variants.
+
+            If 'names' is null or empty, an empty string is returned. If no
+            appropriate localization is found, the first name is returned.
+        """
+        if not names:
+            return ''
+
+        if len(names) > 1:
+            for tag in self.name_tags:
+                if tag in names:
+                    return names[tag]
+
+        # Nothing? Return any of the other names as a default.
+        return next(iter(names.values()))
+
+
+    @staticmethod
+    def from_accept_languages(langstr: str) -> 'Locales':
+        """ Create a localization object from a language list in the
+            format of HTTP accept-languages header.
+
+            The functions tries to be forgiving of format errors by first splitting
+            the string into comma-separated parts and then parsing each
+            description separately. Badly formatted parts are then ignored.
+        """
+        # split string into languages
+        candidates = []
+        for desc in langstr.split(','):
+            m = re.fullmatch(r'\s*([a-z_-]+)(?:;\s*q\s*=\s*([01](?:\.\d+)?))?\s*',
+                             desc, flags=re.I)
+            if m:
+                candidates.append((m[1], float(m[2] or 1.0)))
+
+        # sort the results by the weight of each language (preserving order).
+        candidates.sort(reverse=True, key=lambda e: e[1])
+
+        # If a language has a region variant, also add the language without
+        # variant but only if it isn't already in the list to not mess up the weight.
+        languages = []
+        for lid, _ in candidates:
+            languages.append(lid)
+            parts = lid.split('-', 1)
+            if len(parts) > 1 and all(c[0] != parts[0] for c in candidates):
+                languages.append(parts[0])
+
+        return Locales(languages)

nominatim/api/result_formatting.py

@@ -7,11 +7,11 @@
 """
 Helper classes and functions for formating results into API responses.
 """
-from typing import Type, TypeVar, Dict, List, Callable, Any
+from typing import Type, TypeVar, Dict, List, Callable, Any, Mapping
 from collections import defaultdict
 
 T = TypeVar('T') # pylint: disable=invalid-name
-FormatFunc = Callable[[T], str]
+FormatFunc = Callable[[T, Mapping[str, Any]], str]
 
 
 class FormatDispatcher:
@@ -47,10 +47,10 @@ class FormatDispatcher:
         return fmt in self.format_functions[result_type]
 
 
-    def format_result(self, result: Any, fmt: str) -> str:
+    def format_result(self, result: Any, fmt: str, options: Mapping[str, Any]) -> str:
         """ Convert the given result into a string using the given format.
 
             The format is expected to be in the list returned by
             `list_formats()`.
         """
-        return self.format_functions[type(result)][fmt](result)
+        return self.format_functions[type(result)][fmt](result, options)

nominatim/api/v1/classtypes.py

@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Hard-coded information about tag catagories.
+
+These tables have been copied verbatim from the old PHP code. For future
+version a more flexible formatting is required.
+"""
+
+ICONS = {
+    ('boundary', 'administrative'): 'poi_boundary_administrative',
+    ('place', 'city'): 'poi_place_city',
+    ('place', 'town'): 'poi_place_town',
+    ('place', 'village'): 'poi_place_village',
+    ('place', 'hamlet'): 'poi_place_village',
+    ('place', 'suburb'): 'poi_place_village',
+    ('place', 'locality'): 'poi_place_village',
+    ('place', 'airport'): 'transport_airport2',
+    ('aeroway', 'aerodrome'): 'transport_airport2',
+    ('railway', 'station'): 'transport_train_station2',
+    ('amenity', 'place_of_worship'): 'place_of_worship_unknown3',
+    ('amenity', 'pub'): 'food_pub',
+    ('amenity', 'bar'): 'food_bar',
+    ('amenity', 'university'): 'education_university',
+    ('tourism', 'museum'): 'tourist_museum',
+    ('amenity', 'arts_centre'): 'tourist_art_gallery2',
+    ('tourism', 'zoo'): 'tourist_zoo',
+    ('tourism', 'theme_park'): 'poi_point_of_interest',
+    ('tourism', 'attraction'): 'poi_point_of_interest',
+    ('leisure', 'golf_course'): 'sport_golf',
+    ('historic', 'castle'): 'tourist_castle',
+    ('amenity', 'hospital'): 'health_hospital',
+    ('amenity', 'school'): 'education_school',
+    ('amenity', 'theatre'): 'tourist_theatre',
+    ('amenity', 'library'): 'amenity_library',
+    ('amenity', 'fire_station'): 'amenity_firestation3',
+    ('amenity', 'police'): 'amenity_police2',
+    ('amenity', 'bank'): 'money_bank2',
+    ('amenity', 'post_office'): 'amenity_post_office',
+    ('tourism', 'hotel'): 'accommodation_hotel2',
+    ('amenity', 'cinema'): 'tourist_cinema',
+    ('tourism', 'artwork'): 'tourist_art_gallery2',
+    ('historic', 'archaeological_site'): 'tourist_archaeological2',
+    ('amenity', 'doctors'): 'health_doctors',
+    ('leisure', 'sports_centre'): 'sport_leisure_centre',
+    ('leisure', 'swimming_pool'): 'sport_swimming_outdoor',
+    ('shop', 'supermarket'): 'shopping_supermarket',
+    ('shop', 'convenience'): 'shopping_convenience',
+    ('amenity', 'restaurant'): 'food_restaurant',
+    ('amenity', 'fast_food'): 'food_fastfood',
+    ('amenity', 'cafe'): 'food_cafe',
+    ('tourism', 'guest_house'): 'accommodation_bed_and_breakfast',
+    ('amenity', 'pharmacy'): 'health_pharmacy_dispensing',
+    ('amenity', 'fuel'): 'transport_fuel',
+    ('natural', 'peak'): 'poi_peak',
+    ('natural', 'wood'): 'landuse_coniferous_and_deciduous',
+    ('shop', 'bicycle'): 'shopping_bicycle',
+    ('shop', 'clothes'): 'shopping_clothes',
+    ('shop', 'hairdresser'): 'shopping_hairdresser',
+    ('shop', 'doityourself'): 'shopping_diy',
+    ('shop', 'estate_agent'): 'shopping_estateagent2',
+    ('shop', 'car'): 'shopping_car',
+    ('shop', 'garden_centre'): 'shopping_garden_centre',
+    ('shop', 'car_repair'): 'shopping_car_repair',
+    ('shop', 'bakery'): 'shopping_bakery',
+    ('shop', 'butcher'): 'shopping_butcher',
+    ('shop', 'apparel'): 'shopping_clothes',
+    ('shop', 'laundry'): 'shopping_laundrette',
+    ('shop', 'beverages'): 'shopping_alcohol',
+    ('shop', 'alcohol'): 'shopping_alcohol',
+    ('shop', 'optician'): 'health_opticians',
+    ('shop', 'chemist'): 'health_pharmacy',
+    ('shop', 'gallery'): 'tourist_art_gallery2',
+    ('shop', 'jewelry'): 'shopping_jewelry',
+    ('tourism', 'information'): 'amenity_information',
+    ('historic', 'ruins'): 'tourist_ruin',
+    ('amenity', 'college'): 'education_school',
+    ('historic', 'monument'): 'tourist_monument',
+    ('historic', 'memorial'): 'tourist_monument',
+    ('historic', 'mine'): 'poi_mine',
+    ('tourism', 'caravan_site'): 'accommodation_caravan_park',
+    ('amenity', 'bus_station'): 'transport_bus_station',
+    ('amenity', 'atm'): 'money_atm2',
+    ('tourism', 'viewpoint'): 'tourist_view_point',
+    ('tourism', 'guesthouse'): 'accommodation_bed_and_breakfast',
+    ('railway', 'tram'): 'transport_tram_stop',
+    ('amenity', 'courthouse'): 'amenity_court',
+    ('amenity', 'recycling'): 'amenity_recycling',
+    ('amenity', 'dentist'): 'health_dentist',
+    ('natural', 'beach'): 'tourist_beach',
+    ('railway', 'tram_stop'): 'transport_tram_stop',
+    ('amenity', 'prison'): 'amenity_prison',
+    ('highway', 'bus_stop'): 'transport_bus_stop2'
+}

nominatim/api/v1/format.py

@@ -7,22 +7,26 @@
 """
 Output formatters for API version v1.
 """
+from typing import Mapping, Any
+import collections
+
+import nominatim.api as napi
 from nominatim.api.result_formatting import FormatDispatcher
-from nominatim.api import StatusResult
+from nominatim.api.v1.classtypes import ICONS
 from nominatim.utils.json_writer import JsonWriter
 
 dispatch = FormatDispatcher()
 
-@dispatch.format_func(StatusResult, 'text')
-def _format_status_text(result: StatusResult) -> str:
+@dispatch.format_func(napi.StatusResult, 'text')
+def _format_status_text(result: napi.StatusResult, _: Mapping[str, Any]) -> str:
     if result.status:
         return f"ERROR: {result.message}"
 
     return 'OK'
 
 
-@dispatch.format_func(StatusResult, 'json')
-def _format_status_json(result: StatusResult) -> str:
+@dispatch.format_func(napi.StatusResult, 'json')
+def _format_status_json(result: napi.StatusResult, _: Mapping[str, Any]) -> str:
     out = JsonWriter()
 
     out.start_object()\
@@ -35,3 +39,125 @@ def _format_status_json(result: StatusResult) -> str:
        .end_object()
 
     return out()
+
+
+def _add_address_row(writer: JsonWriter, row: napi.AddressLine,
+                     locales: napi.Locales) -> None:
+    writer.start_object()\
+            .keyval('localname', locales.display_name(row.names))\
+            .keyval('place_id', row.place_id)
+
+    if row.osm_object is not None:
+        writer.keyval('osm_id', row.osm_object[1])\
+              .keyval('osm_type', row.osm_object[0])
+
+    if row.extratags:
+        writer.keyval_not_none('place_type', row.extratags.get('place_type'))
+
+    writer.keyval('class', row.category[0])\
+          .keyval('type', row.category[1])\
+          .keyval_not_none('admin_level', row.admin_level)\
+          .keyval('rank_address', row.rank_address)\
+          .keyval('distance', row.distance)\
+          .keyval('isaddress', row.isaddress)\
+        .end_object()
+
+
+def _add_address_rows(writer: JsonWriter, section: str, rows: napi.AddressLines,
+                      locales: napi.Locales) -> None:
+    writer.key(section).start_array()
+    for row in rows:
+        _add_address_row(writer, row, locales)
+        writer.next()
+    writer.end_array().next()
+
+
+def _add_parent_rows_grouped(writer: JsonWriter, rows: napi.AddressLines,
+                             locales: napi.Locales) -> None:
+    # group by category type
+    data = collections.defaultdict(list)
+    for row in rows:
+        sub = JsonWriter()
+        _add_address_row(sub, row, locales)
+        data[row.category[1]].append(sub())
+
+    writer.key('hierarchy').start_object()
+    for group, grouped in data.items():
+        writer.key(group).start_array()
+        grouped.sort() # sorts alphabetically by local name
+        for line in grouped:
+            writer.raw(line).next()
+        writer.end_array().next()
+
+    writer.end_object().next()
+
+
+@dispatch.format_func(napi.SearchResult, 'details-json')
+def _format_search_json(result: napi.SearchResult, options: Mapping[str, Any]) -> str:
+    locales = options.get('locales', napi.Locales())
+    geom = result.geometry.get('geojson')
+    centroid = result.centroid_as_geojson()
+
+    out = JsonWriter()
+    out.start_object()\
+         .keyval('place_id', result.place_id)\
+         .keyval('parent_place_id', result.parent_place_id)
+
+    if result.osm_object is not None:
+        out.keyval('osm_type', result.osm_object[0])\
+           .keyval('osm_id', result.osm_object[1])
+
+    out.keyval('category', result.category[0])\
+         .keyval('type', result.category[1])\
+         .keyval('admin_level', result.admin_level)\
+         .keyval('localname', locales.display_name(result.names))\
+         .keyval('names', result.names or [])\
+         .keyval('addresstags', result.address or [])\
+         .keyval('housenumber', result.housenumber)\
+         .keyval('calculated_postcode', result.postcode)\
+         .keyval('country_code', result.country_code)\
+         .keyval_not_none('indexed_date', result.indexed_date, lambda v: v.isoformat())\
+         .keyval('importance', result.importance)\
+         .keyval('calculated_importance', result.calculated_importance())\
+         .keyval('extratags', result.extratags or [])\
+         .keyval('calculated_wikipedia', result.wikipedia)\
+         .keyval('rank_address', result.rank_address)\
+         .keyval('rank_search', result.rank_search)\
+         .keyval('isarea', 'Polygon' in (geom or result.geometry.get('type') or ''))\
+         .key('centroid').raw(centroid).next()\
+         .key('geometry').raw(geom or centroid).next()
+
+    if options.get('icon_base_url', None):
+        icon = ICONS.get(result.category)
+        if icon:
+            out.keyval('icon', f"{options['icon_base_url']}/{icon}.p.20.png")
+
+    if result.address_rows is not None:
+        _add_address_rows(out, 'address', result.address_rows, locales)
+
+    if result.linked_rows is not None:
+        _add_address_rows(out, 'linked_places', result.linked_rows, locales)
+
+    if result.name_keywords is not None or result.address_keywords is not None:
+        out.key('keywords').start_object()
+
+        for sec, klist in (('name', result.name_keywords), ('address', result.address_keywords)):
+            out.key(sec).start_array()
+            for word in (klist or []):
+                out.start_object()\
+                     .keyval('id', word.word_id)\
+                     .keyval('token', word.word_token)\
+                   .end_object().next()
+            out.end_array().next()
+
+        out.end_object().next()
+
+    if result.parented_rows is not None:
+        if options.get('group_hierarchy', False):
+            _add_parent_rows_grouped(out, result.parented_rows, locales)
+        else:
+            _add_address_rows(out, 'hierarchy', result.parented_rows, locales)
+
+    out.end_object()
+
+    return out()

nominatim/api/v1/server_glue.py

@@ -143,7 +143,7 @@ async def status_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A
     else:
         status_code = 200
 
-    return params.build_response(formatting.format_result(result, fmt), fmt,
+    return params.build_response(formatting.format_result(result, fmt, {}), fmt,
                                  status=status_code)
 
 EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any]

nominatim/clicmd/api.py

@@ -10,11 +10,13 @@ Subcommand definitions for API calls from the command line.
 from typing import Mapping, Dict
 import argparse
 import logging
+import json
+import sys
 
 from nominatim.tools.exec_utils import run_api_script
 from nominatim.errors import UsageError
 from nominatim.clicmd.args import NominatimArgs
-from nominatim.api import NominatimAPI, StatusResult
+import nominatim.api as napi
 import nominatim.api.v1 as api_output
 
 # Do not repeat documentation of subcommand classes.
@@ -38,15 +40,6 @@ EXTRADATA_PARAMS = (
     ('namedetails', 'Include a list of alternative names')
 )
 
-DETAILS_SWITCHES = (
-    ('addressdetails', 'Include a breakdown of the address into elements'),
-    ('keywords', 'Include a list of name keywords and address keywords'),
-    ('linkedplaces', 'Include a details of places that are linked with this one'),
-    ('hierarchy', 'Include details of places lower in the address hierarchy'),
-    ('group_hierarchy', 'Group the places by type'),
-    ('polygon_geojson', 'Include geometry of result')
-)
-
 def _add_api_output_arguments(parser: argparse.ArgumentParser) -> None:
     group = parser.add_argument_group('Output arguments')
     group.add_argument('--format', default='jsonv2',
@@ -240,29 +233,66 @@ class APIDetails:
                                  "of the same object."))
 
         group = parser.add_argument_group('Output arguments')
-        for name, desc in DETAILS_SWITCHES:
-            group.add_argument('--' + name, action='store_true', help=desc)
+        group.add_argument('--addressdetails', action='store_true',
+                           help='Include a breakdown of the address into elements')
+        group.add_argument('--keywords', action='store_true',
+                           help='Include a list of name keywords and address keywords')
+        group.add_argument('--linkedplaces', action='store_true',
+                           help='Include a details of places that are linked with this one')
+        group.add_argument('--hierarchy', action='store_true',
+                           help='Include details of places lower in the address hierarchy')
+        group.add_argument('--group_hierarchy', action='store_true',
+                           help='Group the places by type')
+        group.add_argument('--polygon_geojson', action='store_true',
+                           help='Include geometry of result')
         group.add_argument('--lang', '--accept-language', metavar='LANGS',
                            help='Preferred language order for presenting search results')
 
 
     def run(self, args: NominatimArgs) -> int:
+        place: napi.PlaceRef
         if args.node:
-            params = dict(osmtype='N', osmid=args.node)
+            place = napi.OsmID('N', args.node, args.object_class)
         elif args.way:
-            params = dict(osmtype='W', osmid=args.way)
+            place = napi.OsmID('W', args.way, args.object_class)
         elif args.relation:
-            params = dict(osmtype='R', osmid=args.relation)
+            place = napi.OsmID('R', args.relation, args.object_class)
         else:
-            params = dict(place_id=args.place_id)
-        if args.object_class:
-            params['class'] = args.object_class
-        for name, _ in DETAILS_SWITCHES:
-            params[name] = '1' if getattr(args, name) else '0'
-        if args.lang:
-            params['accept-language'] = args.lang
+            assert args.place_id is not None
+            place = napi.PlaceID(args.place_id)
 
-        return _run_api('details', args, params)
+        api = napi.NominatimAPI(args.project_dir)
+
+        details = napi.LookupDetails(address_details=args.addressdetails,
+                                     linked_places=args.linkedplaces,
+                                     parented_places=args.hierarchy,
+                                     keywords=args.keywords)
+        if args.polygon_geojson:
+            details.geometry_output = napi.GeometryFormat.GEOJSON
+
+        if args.lang:
+            locales = napi.Locales.from_accept_languages(args.lang)
+        elif api.config.DEFAULT_LANGUAGE:
+            locales = napi.Locales.from_accept_languages(api.config.DEFAULT_LANGUAGE)
+        else:
+            locales = napi.Locales()
+
+        result = api.lookup(place, details)
+
+        if result:
+            output = api_output.format_result(
+                        result,
+                        'details-json',
+                        {'locales': locales,
+                         'group_hierarchy': args.group_hierarchy})
+            # reformat the result, so it is pretty-printed
+            json.dump(json.loads(output), sys.stdout, indent=4)
+            sys.stdout.write('\n')
+
+            return 0
+
+        LOG.error("Object not found in database.")
+        return 42
 
 
 class APIStatus:
@@ -276,13 +306,13 @@ class APIStatus:
     """
 
     def add_args(self, parser: argparse.ArgumentParser) -> None:
-        formats = api_output.list_formats(StatusResult)
+        formats = api_output.list_formats(napi.StatusResult)
         group = parser.add_argument_group('API parameters')
         group.add_argument('--format', default=formats[0], choices=formats,
                            help='Format of result')
 
 
     def run(self, args: NominatimArgs) -> int:
-        status = NominatimAPI(args.project_dir).status()
-        print(api_output.format_result(status, args.format))
+        status = napi.NominatimAPI(args.project_dir).status()
+        print(api_output.format_result(status, args.format, {}))
         return 0

nominatim/clicmd/args.py

@@ -168,6 +168,11 @@ class NominatimArgs:
 
     # Arguments to 'details'
     object_class: Optional[str]
+    linkedplaces: bool
+    hierarchy: bool
+    keywords: bool
+    polygon_geojson: bool
+    group_hierarchy: bool
 
 
     def osm2pgsql_options(self, default_cache: int,