Merge branch 'osm-search:master' into check-database-on-frozen-database

.github/actions/build-nominatim/action.yml

@@ -25,7 +25,7 @@ runs:
           shell: bash
         - name: Install${{ matrix.flavour }} prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson
             if [ "$FLAVOUR" == "oldstuff" ]; then
                 pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg
             else

.github/workflows/ci-tests.yml

@@ -113,7 +113,7 @@ jobs:
               if: matrix.flavour == 'oldstuff'
 
             - name: Install Python webservers
-              run: pip3 install falcon sanic sanic-testing sanic-cors starlette
+              run: pip3 install falcon starlette
 
             - name: Install latest pylint
               run: pip3 install -U pylint asgi_lifespan
@@ -250,6 +250,9 @@ jobs:
             - name: Prepare import environment
               run: |
                   mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
+                  mv Nominatim/settings/flex-base.lua flex-base.lua
+                  mv Nominatim/settings/import-extratags.lua import-extratags.lua
+                  mv Nominatim/settings/taginfo.lua taginfo.lua
                   rm -rf Nominatim
                   mkdir data-env-reverse
               working-directory: /home/nominatim
@@ -258,6 +261,10 @@ jobs:
               run: nominatim --version
               working-directory: /home/nominatim/nominatim-project
 
+            - name: Print taginfo
+              run: lua taginfo.lua
+              working-directory: /home/nominatim
+
             - name: Collect host OS information
               run: nominatim admin --collect-os-info
               working-directory: /home/nominatim/nominatim-project

docs/admin/Installation.md

@@ -67,9 +67,8 @@ For running the experimental Python frontend:
 
   * one of the following web frameworks:
     * [falcon](https://falconframework.org/) (3.0+)
-    * [sanic](https://sanic.dev) and (optionally) [sanic-cors](https://github.com/ashleysommer/sanic-cors)
     * [starlette](https://www.starlette.io/)
-  * [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks)
+  * [uvicorn](https://www.uvicorn.org/)
 
 For dependencies for running tests and building documentation, see
 the [Development section](../develop/Development-Environment.md).

docs/develop/Development-Environment.md

@@ -41,7 +41,6 @@ It has the following additional requirements:
 For testing the Python search frontend, you need to install extra dependencies
 depending on your choice of webserver framework:
 
-* [sanic-testing](https://sanic.dev/en/plugins/sanic-testing/getting-started.html) (sanic only)
 * [httpx](https://www.python-httpx.org/) (starlette only)
 * [asgi-lifespan](https://github.com/florimondmanca/asgi-lifespan) (starlette only)
 
@@ -66,7 +65,7 @@ sudo apt install php-cgi phpunit php-codesniffer \
 pip3 install --user behave mkdocs mkdocstrings pytest pytest-asyncio pylint \
                     mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \
                     types-ujson types-requests types-Pygments typing-extensions\
-                    sanic-testing httpx asgi-lifespan
+                    httpx asgi-lifespan
 ```
 
 The `mkdocs` executable will be located in `.local/bin`. You may have to add

nominatim/api/logging.py

@@ -178,7 +178,7 @@ class HTMLLogger(BaseLogger):
             self._write(f"rank={res.rank_address}, ")
             self._write(f"osm={format_osm(res.osm_object)}, ")
             self._write(f'cc={res.country_code}, ')
-            self._write(f'importance={res.importance or -1:.5f})</dd>')
+            self._write(f'importance={res.importance or float("nan"):.5f})</dd>')
             total += 1
         self._write(f'</dl><b>TOTAL:</b> {total}</p>')
 
@@ -196,7 +196,7 @@ class HTMLLogger(BaseLogger):
 
     def _python_var(self, var: Any) -> str:
         if CODE_HIGHLIGHT:
-            fmt = highlight(repr(var), PythonLexer(), HtmlFormatter(nowrap=True))
+            fmt = highlight(str(var), PythonLexer(), HtmlFormatter(nowrap=True))
             return f'<div class="highlight"><code class="lang-python">{fmt}</code></div>'
 
         return f'<code class="lang-python">{str(var)}</code>'

nominatim/api/search/db_search_builder.py

@@ -141,12 +141,14 @@ class SearchBuilder:
             yield dbs.CountrySearch(sdata)
 
         if sdata.postcodes and (is_category or self.configured_for_postcode):
+            penalty = 0.0 if sdata.countries else 0.1
             if address:
                 sdata.lookups = [dbf.FieldLookup('nameaddress_vector',
                                                  [t.token for r in address
                                                   for t in self.query.get_partials_list(r)],
                                                  'restrict')]
-            yield dbs.PostcodeSearch(0.4, sdata)
+                penalty += 0.2
+            yield dbs.PostcodeSearch(penalty, sdata)
 
 
     def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],

nominatim/api/search/db_searches.py

@@ -403,6 +403,12 @@ class CountrySearch(AbstractSearch):
                                       details: SearchDetails) -> nres.SearchResults:
         """ Look up the country in the fallback country tables.
         """
+        # Avoid the fallback search when this is a more search. Country results
+        # usually are in the first batch of results and it is not possible
+        # to exclude these fallbacks.
+        if details.excluded:
+            return nres.SearchResults()
+
         t = conn.t.country_name
         tgrid = conn.t.country_grid
 
@@ -562,6 +568,8 @@ class PlaceSearch(AbstractSearch):
             sql = sql.where(tsearch.c.country_code.in_(self.countries.values))
 
         if self.postcodes:
+            # if a postcode is given, don't search for state or country level objects
+            sql = sql.where(tsearch.c.address_rank > 9)
             tpc = conn.t.postcode
             if self.expected_count > 1000:
                 # Many results expected. Restrict by postcode.

nominatim/api/search/geocoder.py

@@ -180,7 +180,7 @@ def _dump_searches(searches: List[AbstractSearch], query: QueryStruct,
         return f'{c[0]}^{c[1]}'
 
     for search in searches[start:]:
-        fields = ('name_lookups', 'name_ranking', 'countries', 'housenumbers',
+        fields = ('lookups', 'rankings', 'countries', 'housenumbers',
                   'postcodes', 'qualifier')
         iters = itertools.zip_longest([f"{search.penalty:.3g}"],
                                       *(getattr(search, attr, []) for attr in fields),

nominatim/api/search/icu_tokenizer.py

@@ -153,7 +153,7 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
         """
         log().section('Analyze query (using ICU tokenizer)')
         normalized = list(filter(lambda p: p.text,
-                                 (qmod.Phrase(p.ptype, self.normalizer.transliterate(p.text))
+                                 (qmod.Phrase(p.ptype, self.normalize_text(p.text))
                                   for p in phrases)))
         query = qmod.QueryStruct(normalized)
         log().var_dump('Normalized query', query.source)
@@ -187,6 +187,14 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
         return query
 
 
+    def normalize_text(self, text: str) -> str:
+        """ Bring the given text into a normalized form. That is the
+            standardized form search will work with. All information removed
+            at this stage is inevitably lost.
+        """
+        return cast(str, self.normalizer.transliterate(text))
+
+
     def split_query(self, query: qmod.QueryStruct) -> Tuple[QueryParts, WordDict]:
         """ Transliterate the phrases and split them into tokens.
 
@@ -248,12 +256,11 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
                        and (repl.ttype != qmod.TokenType.HOUSENUMBER
                             or len(tlist.tokens[0].lookup_word) > 4):
                         repl.add_penalty(0.39)
-            elif tlist.ttype == qmod.TokenType.HOUSENUMBER:
+            elif tlist.ttype == qmod.TokenType.HOUSENUMBER \
+                 and len(tlist.tokens[0].lookup_word) <= 3:
                 if any(c.isdigit() for c in tlist.tokens[0].lookup_word):
                     for repl in node.starting:
-                        if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER \
+                        if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER:
-                           and (repl.ttype != qmod.TokenType.HOUSENUMBER
-                                or len(tlist.tokens[0].lookup_word) <= 3):
                             repl.add_penalty(0.5 - tlist.tokens[0].penalty)
             elif tlist.ttype not in (qmod.TokenType.COUNTRY, qmod.TokenType.PARTIAL):
                 norm = parts[i].normalized

nominatim/api/search/legacy_tokenizer.py

@@ -233,12 +233,11 @@ class LegacyQueryAnalyzer(AbstractQueryAnalyzer):
                        and (repl.ttype != qmod.TokenType.HOUSENUMBER
                             or len(tlist.tokens[0].lookup_word) > 4):
                         repl.add_penalty(0.39)
-            elif tlist.ttype == qmod.TokenType.HOUSENUMBER:
+            elif tlist.ttype == qmod.TokenType.HOUSENUMBER \
+                 and len(tlist.tokens[0].lookup_word) <= 3:
                 if any(c.isdigit() for c in tlist.tokens[0].lookup_word):
                     for repl in node.starting:
-                        if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER \
+                        if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER:
-                           and (repl.ttype != qmod.TokenType.HOUSENUMBER
-                                or len(tlist.tokens[0].lookup_word) <= 3):
                             repl.add_penalty(0.5 - tlist.tokens[0].penalty)
 
 

nominatim/api/search/token_assignment.py

@@ -270,7 +270,12 @@ class _TokenSequence:
             if (base.postcode.start == 0 and self.direction != -1)\
                or (base.postcode.end == query.num_token_slots() and self.direction != 1):
                 log().comment('postcode search')
-                yield dataclasses.replace(base, penalty=self.penalty)
+                # <address>,<postcode> should give preference to address search
+                if base.postcode.start == 0:
+                    penalty = self.penalty
+                else:
+                    penalty = self.penalty + 0.1
+                yield dataclasses.replace(base, penalty=penalty)
 
         # Postcode or country-only search
         if not base.address:
@@ -278,6 +283,9 @@ class _TokenSequence:
                 log().comment('postcode/country search')
                 yield dataclasses.replace(base, penalty=self.penalty)
         else:
+            # <postcode>,<address> should give preference to postcode search
+            if base.postcode and base.postcode.start == 0:
+                self.penalty += 0.1
             # Use entire first word as name
             if self.direction != -1:
                 log().comment('first word = name')

nominatim/api/types.py

@@ -302,10 +302,11 @@ def format_excluded(ids: Any) -> List[int]:
     else:
         raise UsageError("Parameter 'excluded' needs to be a comma-separated list "
                          "or a Python list of numbers.")
-    if not all(isinstance(i, int) or (isinstance(i, str) and i.isdigit()) for i in plist):
+    if not all(isinstance(i, int) or
+               (isinstance(i, str) and (not i or i.isdigit())) for i in plist):
         raise UsageError("Parameter 'excluded' only takes place IDs.")
 
-    return [int(id) for id in plist if id]
+    return [int(id) for id in plist if id] or [0]
 
 
 def format_categories(categories: List[Tuple[str, str]]) -> List[Tuple[str, str]]:

nominatim/api/v1/helpers.py

@@ -62,13 +62,13 @@ def extend_query_parts(queryparts: Dict[str, Any], details: Dict[str, Any],
     """
     parsed = SearchDetails.from_kwargs(details)
     if parsed.geometry_output != GeometryFormat.NONE:
-        if parsed.geometry_output & GeometryFormat.GEOJSON:
+        if GeometryFormat.GEOJSON in parsed.geometry_output:
             queryparts['polygon_geojson'] = '1'
-        if parsed.geometry_output & GeometryFormat.KML:
+        if GeometryFormat.KML in parsed.geometry_output:
             queryparts['polygon_kml'] = '1'
-        if parsed.geometry_output & GeometryFormat.SVG:
+        if GeometryFormat.SVG in parsed.geometry_output:
             queryparts['polygon_svg'] = '1'
-        if parsed.geometry_output & GeometryFormat.TEXT:
+        if GeometryFormat.TEXT in parsed.geometry_output:
             queryparts['polygon_text'] = '1'
     if parsed.address_details:
         queryparts['addressdetails'] = '1'

nominatim/api/v1/server_glue.py

@@ -185,7 +185,7 @@ class ASGIAdaptor(abc.ABC):
         """ Return the accepted languages.
         """
         return self.get('accept-language')\
-               or self.get_header('http_accept_language')\
+               or self.get_header('accept-language')\
                or self.config().DEFAULT_LANGUAGE
 
 

nominatim/cli.py

@@ -215,7 +215,7 @@ class AdminServe:
         group.add_argument('--server', default='127.0.0.1:8088',
                            help='The address the server will listen to.')
         group.add_argument('--engine', default='php',
-                           choices=('php', 'sanic', 'falcon', 'starlette'),
+                           choices=('php', 'falcon', 'starlette'),
                            help='Webserver framework to run. (default: php)')
 
 
@@ -223,6 +223,7 @@ class AdminServe:
         if args.engine == 'php':
             run_php_server(args.server, args.project_dir / 'website')
         else:
+            import uvicorn # pylint: disable=import-outside-toplevel
             server_info = args.server.split(':', 1)
             host = server_info[0]
             if len(server_info) > 1:
@@ -232,21 +233,10 @@ class AdminServe:
             else:
                 port = 8088
 
-            if args.engine == 'sanic':
+            server_module = importlib.import_module(f'nominatim.server.{args.engine}.server')
-                server_module = importlib.import_module('nominatim.server.sanic.server')
 
-                app = server_module.get_application(args.project_dir)
+            app = server_module.get_application(args.project_dir)
-                app.run(host=host, port=port, debug=True, single_process=True)
+            uvicorn.run(app, host=host, port=port)
-            else:
-                import uvicorn # pylint: disable=import-outside-toplevel
-
-                if args.engine == 'falcon':
-                    server_module = importlib.import_module('nominatim.server.falcon.server')
-                elif args.engine == 'starlette':
-                    server_module = importlib.import_module('nominatim.server.starlette.server')
-
-                app = server_module.get_application(args.project_dir)
-                uvicorn.run(app, host=host, port=port)
 
         return 0
 

nominatim/server/sanic/server.py

@@ -1,78 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# This file is part of Nominatim. (https://nominatim.org)
-#
-# Copyright (C) 2023 by the Nominatim developer community.
-# For a full list of authors see the git log.
-"""
-Server implementation using the sanic webserver framework.
-"""
-from typing import Any, Optional, Mapping, Callable, cast, Coroutine
-from pathlib import Path
-
-from sanic import Request, HTTPResponse, Sanic
-from sanic.exceptions import SanicException
-from sanic.response import text as TextResponse
-
-from nominatim.api import NominatimAPIAsync
-import nominatim.api.v1 as api_impl
-from nominatim.config import Configuration
-
-class ParamWrapper(api_impl.ASGIAdaptor):
-    """ Adaptor class for server glue to Sanic framework.
-    """
-
-    def __init__(self, request: Request) -> None:
-        self.request = request
-
-
-    def get(self, name: str, default: Optional[str] = None) -> Optional[str]:
-        return cast(Optional[str], self.request.args.get(name, default))
-
-
-    def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
-        return cast(Optional[str], self.request.headers.get(name, default))
-
-
-    def error(self, msg: str, status: int = 400) -> SanicException:
-        exception = SanicException(msg, status_code=status)
-
-        return exception
-
-
-    def create_response(self, status: int, output: str) -> HTTPResponse:
-        return TextResponse(output, status=status, content_type=self.content_type)
-
-
-    def config(self) -> Configuration:
-        return cast(Configuration, self.request.app.ctx.api.config)
-
-
-def _wrap_endpoint(func: api_impl.EndpointFunc)\
-       -> Callable[[Request], Coroutine[Any, Any, HTTPResponse]]:
-    async def _callback(request: Request) -> HTTPResponse:
-        return cast(HTTPResponse, await func(request.app.ctx.api, ParamWrapper(request)))
-
-    return _callback
-
-
-def get_application(project_dir: Path,
-                    environ: Optional[Mapping[str, str]] = None) -> Sanic:
-    """ Create a Nominatim sanic ASGI application.
-    """
-    app = Sanic("NominatimInstance")
-
-    app.ctx.api = NominatimAPIAsync(project_dir, environ)
-
-    if app.ctx.api.config.get_bool('CORS_NOACCESSCONTROL'):
-        from sanic_cors import CORS # pylint: disable=import-outside-toplevel
-        CORS(app)
-
-    legacy_urls = app.ctx.api.config.get_bool('SERVE_LEGACY_URLS')
-    for name, func in api_impl.ROUTES:
-        endpoint = _wrap_endpoint(func)
-        app.add_route(endpoint, f"/{name}", name=f"v1_{name}_simple")
-        if legacy_urls:
-            app.add_route(endpoint, f"/{name}.php", name=f"v1_{name}_legacy")
-
-    return app

settings/flex-base.lua

@@ -11,6 +11,11 @@ local ADDRESS_TAGS = nil
 local SAVE_EXTRA_MAINS = false
 local POSTCODE_FALLBACK = true
 
+-- tables required for taginfo
+module.TAGINFO_MAIN = {keys = {}, delete_tags = {}}
+module.TAGINFO_NAME_KEYS = {}
+module.TAGINFO_ADDRESS_KEYS = {}
+
 
 -- The single place table.
 local place_table = osm2pgsql.define_table{
@@ -372,6 +377,17 @@ function module.tag_group(data)
     end
 end
 
+-- Returns prefix part of the keys, and reject suffix matching keys
+local function process_key(key)
+    if key:sub(1, 1) == '*' then
+        return nil
+    end
+    if key:sub(#key, #key) == '*' then
+        return key:sub(1, #key - 2)
+    end
+    return key
+end
+
 -- Process functions for all data types
 function module.process_node(object)
 
@@ -465,14 +481,29 @@ function module.set_prefilters(data)
     PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
     PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
                                   tags = data.extra_tags}
+    module.TAGINFO_MAIN.delete_tags = data.delete_tags
 end
 
 function module.set_main_tags(data)
     MAIN_KEYS = data
+    local keys = {}
+    for k, _ in pairs(data) do
+        table.insert(keys, k)
+    end
+    module.TAGINFO_MAIN.keys = keys
 end
 
 function module.set_name_tags(data)
     NAMES = module.tag_group(data)
+
+    for _, lst in pairs(data) do
+        for _, k in ipairs(lst) do
+            local key = process_key(k)
+            if key ~= nil then
+                module.TAGINFO_NAME_KEYS[key] = true
+            end
+        end
+    end
 end
 
 function module.set_address_tags(data)
@@ -480,8 +511,18 @@ function module.set_address_tags(data)
         POSTCODE_FALLBACK = data.postcode_fallback
         data.postcode_fallback = nil
     end
-
     ADDRESS_TAGS = module.tag_group(data)
+
+    for _, lst in pairs(data) do
+        if lst ~= nil then
+            for _, k in ipairs(lst) do
+                local key = process_key(k)
+                if key ~= nil then
+                    module.TAGINFO_ADDRESS_KEYS[key] = true
+                end
+            end
+        end
+    end
 end
 
 function module.set_unused_handling(data)

settings/import-extratags.lua

@@ -7,7 +7,6 @@ flex.set_main_tags{
     historic = 'always',
     military = 'always',
     natural = 'named',
-    landuse = 'named',
     highway = {'always',
                street_lamp = 'named',
                traffic_signals = 'named',

settings/import-full.lua

@@ -7,7 +7,6 @@ flex.set_main_tags{
     historic = 'always',
     military = 'always',
     natural = 'named',
-    landuse = 'named',
     highway = {'always',
                street_lamp = 'named',
                traffic_signals = 'named',

settings/taginfo.lua

@@ -0,0 +1,74 @@
+-- Prints taginfo project description in the standard output
+--
+
+-- create fake "osm2pgsql" table for flex-base, originally created by the main C++ program
+osm2pgsql = {}
+function osm2pgsql.define_table(...) end
+
+-- provide path to flex-style lua file
+flex = require('import-extratags')
+local json = require ('dkjson')
+
+
+------------ helper functions ---------------------
+
+function get_key_description(key, description)
+    local desc = {}
+    desc.key = key
+    desc.description = description
+    set_keyorder(desc, {'key', 'description'})
+    return desc
+end
+
+-- Sets the key order for the resulting JSON table
+function set_keyorder(table, order)
+    setmetatable(table, {
+        __jsonorder = order
+    })
+end
+
+
+-- Prints the collected tags in the required format in JSON
+function print_taginfo()
+    local tags = {}
+
+    for _, k in ipairs(flex.TAGINFO_MAIN.keys) do
+        local desc = get_key_description(k, 'POI/feature in the search database')
+        if flex.TAGINFO_MAIN.delete_tags[k] ~= nil then
+            desc.description = string.format('%s(except for values: %s).', desc.description,
+                                table.concat(flex.TAGINFO_MAIN.delete_tags[k], ', '))
+        end
+        table.insert(tags, desc)
+    end
+
+    for k, _ in pairs(flex.TAGINFO_NAME_KEYS) do
+        local desc = get_key_description(k, 'Searchable name of the place.')
+        table.insert(tags, desc)
+    end
+    for k, _ in pairs(flex.TAGINFO_ADDRESS_KEYS) do
+        local desc = get_key_description(k, 'Used to determine the address of a place.')
+        table.insert(tags, desc)
+    end
+
+    local format = {
+        data_format = 1,
+        data_url = 'https://nominatim.openstreetmap.org/taginfo.json',
+        project = {
+            name = 'Nominatim',
+            description = 'OSM search engine.',
+            project_url = 'https://nominatim.openstreetmap.org',
+            doc_url = 'https://nominatim.org/release-docs/develop/',
+            contact_name = 'Sarah Hoffmann',
+            contact_email = 'lonvia@denofr.de'
+        }
+    }
+    format.tags = tags
+
+    set_keyorder(format, {'data_format', 'data_url', 'project', 'tags'})
+    set_keyorder(format.project, {'name', 'description', 'project_url', 'doc_url',
+                    'contact_name', 'contact_email'})
+
+    print(json.encode(format))
+end
+
+print_taginfo()

test/bdd/steps/nominatim_environment.py

@@ -350,20 +350,6 @@ class NominatimEnvironment:
         return _request
 
 
-    def create_api_request_func_sanic(self):
-        import nominatim.server.sanic.server
-
-        async def _request(endpoint, params, project_dir, environ, http_headers):
-            app = nominatim.server.sanic.server.get_application(project_dir, environ)
-
-            _, response = await app.asgi_client.get(f"/{endpoint}", params=params,
-                                                    headers=http_headers)
-
-            return response.text, response.status_code
-
-        return _request
-
-
     def create_api_request_func_falcon(self):
         import nominatim.server.falcon.server
         import falcon.testing

test/python/api/search/test_icu_query_analyzer.py

@@ -118,10 +118,10 @@ async def test_penalty_postcodes_and_housenumbers(conn, term, order):
 
     assert query.num_token_slots() == 1
 
-    torder = [(tl.tokens[0].penalty, tl.ttype) for tl in query.nodes[0].starting]
+    torder = [(tl.tokens[0].penalty, tl.ttype.name) for tl in query.nodes[0].starting]
     torder.sort()
 
-    assert [t[1] for t in torder] == [TokenType[o] for o in order]
+    assert [t[1] for t in torder] == order
 
 @pytest.mark.asyncio
 async def test_category_words_only_at_beginning(conn):

test/python/api/search/test_legacy_query_analyzer.py

@@ -195,11 +195,10 @@ async def test_penalty_postcodes_and_housenumbers(conn, term, order):
 
     assert query.num_token_slots() == 1
 
-    torder = [(tl.tokens[0].penalty, tl.ttype) for tl in query.nodes[0].starting]
+    torder = [(tl.tokens[0].penalty, tl.ttype.name) for tl in query.nodes[0].starting]
-    print(query.nodes[0].starting)
     torder.sort()
 
-    assert [t[1] for t in torder] == [TokenType[o] for o in order]
+    assert [t[1] for t in torder] == order
 
 
 @pytest.mark.asyncio

test/python/api/search/test_token_assignment.py

@@ -253,7 +253,7 @@ def test_postcode_with_designation():
                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 
     check_assignments(yield_token_assignments(q),
-                      TokenAssignment(name=TokenRange(1, 2),
+                      TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
                                       postcode=TokenRange(0, 1)),
                       TokenAssignment(postcode=TokenRange(0, 1),
                                       address=[TokenRange(1, 2)]))
@@ -266,7 +266,7 @@ def test_postcode_with_designation_backwards():
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),
                                       postcode=TokenRange(1, 2)),
-                      TokenAssignment(postcode=TokenRange(1, 2),
+                      TokenAssignment(penalty=0.1, postcode=TokenRange(1, 2),
                                       address=[TokenRange(0, 1)]))
 
 

test/python/api/test_server_glue_v1.py

@@ -123,7 +123,7 @@ def test_accepted_languages_from_param():
 
 
 def test_accepted_languages_from_header():
-    a = FakeAdaptor(headers={'http_accept_language': 'de'})
+    a = FakeAdaptor(headers={'accept-language': 'de'})
     assert a.get_accepted_languages() == 'de'
 
 
@@ -135,13 +135,13 @@ def test_accepted_languages_from_default(monkeypatch):
 
 def test_accepted_languages_param_over_header():
     a = FakeAdaptor(params={'accept-language': 'de'},
-                    headers={'http_accept_language': 'en'})
+                    headers={'accept-language': 'en'})
     assert a.get_accepted_languages() == 'de'
 
 
 def test_accepted_languages_header_over_default(monkeypatch):
     monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'en')
-    a = FakeAdaptor(headers={'http_accept_language': 'de'})
+    a = FakeAdaptor(headers={'accept-language': 'de'})
     assert a.get_accepted_languages() == 'de'
 
 
@@ -197,14 +197,14 @@ def test_raise_error_during_debug():
     loglib.log().section('Ongoing')
 
     with pytest.raises(FakeError) as excinfo:
-        a.raise_error('bad state')
+        a.raise_error('badstate')
 
     content = ET.fromstring(excinfo.value.msg)
 
     assert content.tag == 'html'
 
     assert '>Ongoing<' in excinfo.value.msg
-    assert 'bad state' in excinfo.value.msg
+    assert 'badstate' in excinfo.value.msg
 
 
 # ASGIAdaptor.build_response

test/python/cli/test_cli.py

@@ -68,15 +68,6 @@ def test_cli_serve_php(cli_call, mock_func_factory):
     assert func.called == 1
 
 
-def test_cli_serve_sanic(cli_call, mock_func_factory):
-    mod = pytest.importorskip("sanic")
-    func = mock_func_factory(mod.Sanic, "run")
-
-    cli_call('serve', '--engine', 'sanic') == 0
-
-    assert func.called == 1
-
-
 def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
     pytest.importorskip("starlette")
     mod = pytest.importorskip("uvicorn")

vagrant/Install-on-Ubuntu-20.sh

@@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
                         postgresql-12-postgis-3 \
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \

vagrant/Install-on-Ubuntu-22.sh

@@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
                         postgresql-server-dev-14 postgresql-14-postgis-3 \
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \