adapt to release 3.6.0

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,16 +7,16 @@ assignees: ''
 
 ---
 
-<!-- Before opening a new feature request, please search through the open issue to check that your request hasn't been reported already. -->
+Before opening a new feature request, please search through the open issue to check that your request hasn't been reported already.
 
 **Is your feature request related to a problem? Please describe.**
-<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
 
 **Describe the solution you'd like**
-<!-- A clear and concise description of what you want to happen. -->
+A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
-<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+A clear and concise description of any alternative solutions or features you've considered.
 
 **Additional context**
-<!-- Add any other context or screenshots about the feature request here. -->
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/report-issues-with-search-results.md

@@ -9,7 +9,7 @@ assignees: ''
 
 ## What did you search for?
 
-<!-- Please try to provide a link to your search. You  can go to https://nominatim.openstreetmap.org and repeat your search there. If you originally found the issue somewhere else, please tell us what software/website you were using. -->
+Please try to provide a link to your search. You  can go to https://nominatim.openstreetmap.org and repeat your search there. If you originally found the issue somewhere else, please tell us what software/website you were using.
 
 ## What result did you get?
 
@@ -17,11 +17,11 @@ assignees: ''
 
 **Is the result in the right place and just named wrongly?** 
 
-<!-- Please tell us the display name you expected. -->
+Please tell us the display name you expected.
 
 **Is the result missing completely?**
 
-<!-- Make sure that the data you are looking for is in OpenStreetMap. Provide a link to the OpenStreetMap object or if you cannot get it, a link to the map on https://openstreetmap.org where you expect the result to be.
+Make sure that the data you are looking for is in OpenStreetMap. Provide a link to the OpenStreetMap object or if you cannot get it, a link to the map on https://openstreetmap.org where you expect the result to be.
 
 To get the link to the OSM object, you can try the following:
 
@@ -30,8 +30,7 @@ To get the link to the OSM object, you can try the following:
  * Click on the question mark on the right side of the map. You get a question cursor. Use it to click on the map where your object is located.
  * Find the object of interest in the list that appears on the left side.
  * Click on the object and report back the URL that the browser shows.
--->
 
 ## Further details
 
-<!-- Anything else we should know about the search. Particularities with addresses in the area etc. -->
+Anything else we should know about the search. Particularities with addresses in the area etc.

.github/ISSUE_TEMPLATE/report-problems-with-the-software.md

@@ -7,13 +7,13 @@ assignees: ''
 
 ---
 
-<!-- Note: if you are installing Nominatim through a docker image, you should report issues with the installation process with the docker repository first. -->
+___Note: if you are installing Nominatim through a docker image, you should report issues with the installation process with the docker repository first.___
 
 **Describe the bug**
-<!-- A clear and concise description of what the bug is. -->
+A clear and concise description of what the bug is.
 
 **To Reproduce**
-<!-- Please describe what you did to get to the issue. -->
+Please describe what you did to get to the issue.
 
 **Software Environment (please complete the following information):**
 - Nominatim version: 
@@ -27,10 +27,5 @@ assignees: ''
 - type and size of disks:
 - bare metal/AWS/other cloud service: 
 
-**Postgresql Configuration:**
-
-<!-- List any configuration items you changed in your postgresql configuration. -->
-
 **Additional context**
-
-<!-- Add any other context about the problem here. -->
+Add any other context about the problem here.

.github/actions/build-nominatim/action.yml

@@ -1,42 +1,28 @@
 name: 'Build Nominatim'
 
-inputs:
-    ubuntu:
-        description: 'Version of Ubuntu to install on'
-        required: false
-        default: '20'
-
 runs:
     using: "composite"
 
     steps:
-        - name: Install prerequisites
-          run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
-            if [ "x$UBUNTUVER" == "x18" ]; then
-                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie
-            else
-                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
-            fi
-          shell: bash
-          env:
-            UBUNTUVER: ${{ inputs.ubuntu }}
-
-        - name: Download dependencies
-          run: |
-              if [ ! -f country_grid.sql.gz ]; then
-                  wget --no-verbose https://www.nominatim.org/data/country_grid.sql.gz
-              fi
-              cp country_grid.sql.gz Nominatim/data/country_osm_grid.sql.gz
+        - name: Install prerequisits
+          run: sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium
           shell: bash
 
         - name: Configure
-          run: mkdir build && cd build && cmake ../Nominatim
+          run: mkdir build && cd build && cmake ..
           shell: bash
 
         - name: Build
           run: |
               make -j2 all
-              sudo make install
+              ./utils/setup.php --setup-website
           shell: bash
           working-directory: build
+
+        - name: Download dependencies
+          run: |
+              if [ ! -f data/country_osm_grid.sql.gz ]; then
+                  wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+              fi
+          shell: bash
+

.github/actions/setup-postgresql/action.yml

@@ -14,10 +14,8 @@ runs:
     steps:
         - name: Remove existing PostgreSQL
           run: |
-              sudo apt-get purge -yq postgresql*
-              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
-
+              sudo apt-get purge -yq postgresql*
           shell: bash
 
         - name: Install PostgreSQL

.github/workflows/ci-tests.yml

@@ -4,40 +4,21 @@ on: [ push, pull_request ]
 
 jobs:
     tests:
+        runs-on: ubuntu-20.04
+
         strategy:
             matrix:
-                ubuntu: [18, 20]
+                postgresql: [9.5, 13]
                 include:
-                    - ubuntu: 18
-                      postgresql: 9.5
+                    - postgresql: 9.5
                       postgis: 2.5
-                      pytest: pytest
-                      php: 7.2
-                    - ubuntu: 20
-                      postgresql: 13
+                    - postgresql: 13
                       postgis: 3
-                      pytest: py.test-3
-                      php: 7.4
-
-        runs-on: ubuntu-${{ matrix.ubuntu }}.04
 
         steps:
             - uses: actions/checkout@v2
               with:
                   submodules: true
-                  path: Nominatim
-
-            - name: Setup PHP
-              uses: shivammathur/setup-php@v2
-              with:
-                  php-version: ${{ matrix.php }}
-                  coverage: xdebug
-                  tools: phpunit, phpcs, composer
-
-            - uses: actions/setup-python@v2
-              with:
-                python-version: 3.6
-              if: matrix.ubuntu == 18
 
             - name: Get Date
               id: get-date
@@ -48,96 +29,39 @@ jobs:
             - uses: actions/cache@v2
               with:
                   path: |
-                     country_grid.sql.gz
-                  key: nominatim-country-data-${{ steps.get-date.outputs.date }}
+                      data/country_osm_grid.sql.gz
+                      monaco-latest.osm.pbf
+                  key: nominatim-data-${{ steps.get-date.outputs.date }}
 
-            - uses: ./Nominatim/.github/actions/setup-postgresql
+            - uses: ./.github/actions/setup-postgresql
               with:
                   postgresql-version: ${{ matrix.postgresql }}
                   postgis-version: ${{ matrix.postgis }}
-
-            - uses: ./Nominatim/.github/actions/build-nominatim
-              with:
-                  ubuntu: ${{ matrix.ubuntu }}
-
-            - name: Install test prerequsites
-              run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage
-              if: matrix.ubuntu == 20
+            - uses: ./.github/actions/build-nominatim
 
             - name: Install test prerequsites
               run: |
-                   pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
-              if: matrix.ubuntu == 18
+                   sudo apt-get install -y -qq php-codesniffer python3-tidylib
+                   sudo pip3 install behave nose
 
             - name: PHP linting
               run: phpcs --report-width=120 .
-              working-directory: Nominatim
-
-            - name: Python linting
-              run: pylint nominatim
-              working-directory: Nominatim
 
             - name: PHP unit tests
-              run: phpunit --coverage-clover ../../coverage-php.xml ./
-              working-directory: Nominatim/test/php
-              if: matrix.ubuntu == 20
-
-            - name: Python unit tests
-              run: $PYTEST --cov=nominatim --cov-report=xml test/python
-              working-directory: Nominatim
-              env:
-                PYTEST: ${{ matrix.pytest }}
+              run: phpunit ./
+              working-directory: test/php
 
             - name: BDD tests
-              run: |
-                  mkdir cov
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov
-                  composer require phpunit/phpcov:7.0.2
-                  vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov
-              working-directory: Nominatim/test/bdd
-              if: matrix.ubuntu == 20
-
-            - name: BDD tests
-              run: |
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
-              working-directory: Nominatim/test/bdd
-              if: matrix.ubuntu == 18
-
-            - name: BDD tests (legacy_icu tokenizer)
-              run: |
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy_icu --format=progress3
-              working-directory: Nominatim/test/bdd
-
-            - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v1
-              with:
-                files: ./Nominatim/coverage*.xml
-                directory: ./
-                name: codecov-umbrella
-                fail_ci_if_error: false
-                path_to_write_report: ./coverage/codecov_report.txt
-                verbose: true
-              if: matrix.ubuntu == 20
+              run: behave -DREMOVE_TEMPLATE=1 --format=progress3 db osm2pgsql
+              working-directory: test/bdd
 
     import:
-        strategy:
-            matrix:
-                ubuntu: [18, 20]
-                include:
-                    - ubuntu: 18
-                      postgresql: 9.5
-                      postgis: 2.5
-                    - ubuntu: 20
-                      postgresql: 13
-                      postgis: 3
-
-        runs-on: ubuntu-${{ matrix.ubuntu }}.04
+        runs-on: ubuntu-20.04
 
         steps:
             - uses: actions/checkout@v2
               with:
                   submodules: true
-                  path: Nominatim
 
             - name: Get Date
               id: get-date
@@ -148,70 +72,49 @@ jobs:
             - uses: actions/cache@v2
               with:
                   path: |
-                     country_grid.sql.gz
-                  key: nominatim-country-data-${{ steps.get-date.outputs.date }}
+                      data/country_osm_grid.sql.gz
+                      monaco-latest.osm.pbf
+                  key: nominatim-data-${{ steps.get-date.outputs.date }}
 
-            - uses: actions/cache@v2
+            - uses: ./.github/actions/setup-postgresql
               with:
-                  path: |
-                     monaco-latest.osm.pbf
-                  key: nominatim-test-data-${{ steps.get-date.outputs.date }}
+                  postgresql-version: 13
+                  postgis-version: 3
+            - uses: ./.github/actions/build-nominatim
 
-            - uses: actions/setup-python@v2
-              with:
-                python-version: 3.6
-              if: matrix.ubuntu == 18
+            - name: Create configuration
+              run: |
+                   echo '<?php' > settings/local.php
+                   echo " @define('CONST_Pyosmium_Binary', '/usr/lib/python3-pyosmium/pyosmium-get-changes');" >> settings/local.php
+              working-directory: build
 
-            - uses: ./Nominatim/.github/actions/setup-postgresql
-              with:
-                  postgresql-version: ${{ matrix.postgresql }}
-                  postgis-version: ${{ matrix.postgis }}
-            - uses: ./Nominatim/.github/actions/build-nominatim
-              with:
-                  ubuntu: ${{ matrix.ubuntu }}
-
-            - name: Clean installation
-              run: rm -rf Nominatim build
-              shell: bash
-
-            - name: Prepare import environment
+            - name: Download import data
               run: |
                   if [ ! -f monaco-latest.osm.pbf ]; then
                       wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf
                   fi
-                  mkdir data-env
-                  cd data-env
               shell: bash
 
             - name: Import
-              run: nominatim import --osm-file ../monaco-latest.osm.pbf
-              shell: bash
-              working-directory: data-env
+              run: php ./utils/setup.php --osm-file ../monaco-latest.osm.pbf --osm2pgsql-cache 500 --all
+              working-directory: build
 
             - name: Import special phrases
-              run: nominatim special-phrases --import-from-wiki
-              working-directory: data-env
+              run: php ./utils/specialphrases.php --wiki-import | psql -d nominatim
+              working-directory: build
 
-            - name: Check full import
-              run: nominatim admin --check-database
-              working-directory: data-env
-
-            - name: Warm up database
-              run: nominatim admin --warm
-              working-directory: data-env
+            - name: Check import
+              run: php ./utils/check_import_finished.php
+              working-directory: build
 
             - name: Run update
               run: |
-                   nominatim replication --init
-                   nominatim replication --once
-              working-directory: data-env
+                   php ./utils/update.php --init-updates
+                   php ./utils/update.php --import-osmosis
+              working-directory: build
 
             - name: Run reverse-only import
-              run : nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only --no-updates
-              working-directory: data-env
-              env:
-                  NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse
-
-            - name: Check reverse import
-              run: nominatim admin --check-database
-              working-directory: data-env
+              run : |
+                  dropdb nominatim
+                  php ./utils/setup.php --osm-file ../monaco-latest.osm.pbf --reverse-only --all
+              working-directory: build

.gitignore

@@ -9,4 +9,3 @@ data/wiki_specialphrases.sql
 data/osmosischange.osc
 
 .vagrant
-data/country_osm_grid.sql.gz

.pylintrc

@@ -1,15 +0,0 @@
-[MASTER]
-
-extension-pkg-whitelist=osmium
-ignored-modules=icu,datrie
-
-[MESSAGES CONTROL]
-
-[TYPECHECK]
-
-# closing added here because it sometimes triggers a false positive with
-# 'with' statements.
-ignored-classes=NominatimArgs,closing
-disable=too-few-public-methods,duplicate-code
-
-good-names=i,x,y,fd,db

CMakeLists.txt

@@ -6,7 +6,7 @@
 #
 #-----------------------------------------------------------------------------
 
-cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 
 
@@ -19,7 +19,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 3)
-set(NOMINATIM_VERSION_MINOR 7)
+set(NOMINATIM_VERSION_MINOR 6)
 set(NOMINATIM_VERSION_PATCH 0)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
@@ -36,7 +36,6 @@ set(BUILD_API on CACHE BOOL "Build everything for the API server")
 set(BUILD_MODULE on CACHE BOOL "Build PostgreSQL module")
 set(BUILD_TESTS on CACHE BOOL "Build test suite")
 set(BUILD_DOCS on CACHE BOOL "Build documentation")
-set(BUILD_MANPAGE on CACHE BOOL "Build Manual Page")
 set(BUILD_OSM2PGSQL on CACHE BOOL "Build osm2pgsql (expert only)")
 
 #-----------------------------------------------------------------------------
@@ -58,11 +57,20 @@ endif()
 
 
 #-----------------------------------------------------------------------------
-#  python (imports/updates only)
+#  python and pyosmium (imports/updates only)
 #-----------------------------------------------------------------------------
 
 if (BUILD_IMPORTER)
-    find_package(PythonInterp 3.6 REQUIRED)
+    find_package(PythonInterp 3)
+
+    find_program(PYOSMIUM pyosmium-get-changes)
+    if (NOT EXISTS "${PYOSMIUM}")
+        set(PYOSMIUM_PATH "")
+            message(WARNING "pyosmium-get-changes not found (required for updates)")
+    else()
+        set(PYOSMIUM_PATH "${PYOSMIUM}")
+        message(STATUS "Using pyosmium-get-changes at ${PYOSMIUM_PATH}")
+    endif()
 endif()
 
 #-----------------------------------------------------------------------------
@@ -78,19 +86,8 @@ if (BUILD_API OR BUILD_IMPORTER)
     # sanity check if PHP binary exists
     if (NOT EXISTS ${PHP_BIN})
         message(FATAL_ERROR "PHP binary not found. Install php or provide location with -DPHP_BIN=/path/php ")
-    else()
-        message (STATUS "Using PHP binary " ${PHP_BIN})
-    endif()
-    if (NOT PHPCGI_BIN)
-        find_program (PHPCGI_BIN php-cgi)
-    endif()
-    # sanity check if PHP binary exists
-    if (NOT EXISTS ${PHPCGI_BIN})
-        message(WARNING "php-cgi binary not found. nominatim tool will not provide query functions.")
-        set (PHPCGI_BIN "")
-    else()
-        message (STATUS "Using php-cgi binary " ${PHPCGI_BIN})
     endif()
+    message (STATUS "Using PHP binary " ${PHP_BIN})
 endif()
 
 #-----------------------------------------------------------------------------
@@ -98,21 +95,70 @@ endif()
 #-----------------------------------------------------------------------------
 
 if (BUILD_IMPORTER)
-   find_file(COUNTRY_GRID_FILE country_osm_grid.sql.gz
-             PATHS ${PROJECT_SOURCE_DIR}/data
-             NO_DEFAULT_PATH
-             DOC "Location of the country grid file."
-            )
+   set(CUSTOMSCRIPTS
+       utils/check_import_finished.php
+       utils/country_languages.php
+       utils/importWikipedia.php
+       utils/export.php
+       utils/query.php
+       utils/setup.php
+       utils/specialphrases.php
+       utils/update.php
+       utils/warm.php
+      )
 
-   if (NOT COUNTRY_GRID_FILE)
-       message(FATAL_ERROR "\nYou need to download the country_osm_grid first:\n"
-                           "    wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz")
-   endif()
-
-   configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl
-                  ${PROJECT_BINARY_DIR}/nominatim)
+   foreach (script_source ${CUSTOMSCRIPTS})
+       configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl
+                      ${PROJECT_BINARY_DIR}/${script_source})
+   endforeach()
 endif()
 
+#-----------------------------------------------------------------------------
+# webserver scripts (API only)
+#-----------------------------------------------------------------------------
+
+if (BUILD_API)
+   set(WEBSITESCRIPTS
+       website/deletable.php
+       website/details.php
+       website/lookup.php
+       website/polygons.php
+       website/reverse.php
+       website/search.php
+       website/status.php
+   )
+
+   foreach (script_source ${WEBSITESCRIPTS})
+       configure_file(${PROJECT_SOURCE_DIR}/cmake/website.tmpl
+                      ${PROJECT_BINARY_DIR}/${script_source})
+   endforeach()
+
+   set(WEBPATHS css images js)
+
+   foreach (wp ${WEBPATHS})
+       execute_process(
+           COMMAND ln -sf ${PROJECT_SOURCE_DIR}/website/${wp} ${PROJECT_BINARY_DIR}/website/
+       )
+   endforeach()
+
+   add_custom_target(serve
+           php -S 127.0.0.1:8088
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
+   )
+
+   add_custom_target(serve-global
+           php -S 0.0.0.0:8088
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
+   )
+endif()
+
+#-----------------------------------------------------------------------------
+# default settings
+#-----------------------------------------------------------------------------
+
+configure_file(${PROJECT_SOURCE_DIR}/settings/defaults.php
+               ${PROJECT_BINARY_DIR}/settings/settings.php)
+
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
@@ -122,60 +168,21 @@ if (BUILD_TESTS)
 
     set(TEST_BDD db osm2pgsql api)
 
-    find_program(PYTHON_BEHAVE behave)
-    find_program(PYLINT NAMES pylint3 pylint)
-    find_program(PYTEST NAMES pytest py.test-3 py.test)
-    find_program(PHPCS phpcs)
-    find_program(PHPUNIT phpunit)
+    foreach (test ${TEST_BDD})
+        add_test(NAME bdd_${test}
+                 COMMAND behave ${test}
+                 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/bdd)
+        set_tests_properties(bdd_${test}
+            PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
+    endforeach()
 
-    if (PYTHON_BEHAVE)
-        message(STATUS "Using Python behave binary ${PYTHON_BEHAVE}")
-        foreach (test ${TEST_BDD})
-            add_test(NAME bdd_${test}
-                     COMMAND ${PYTHON_BEHAVE} ${test}
-                     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/bdd)
-            set_tests_properties(bdd_${test}
-                PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
-        endforeach()
-    else()
-        message(WARNING "behave not found. BDD tests disabled." )
-    endif()
+    add_test(NAME php
+             COMMAND phpunit ./
+             WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/php)
 
-    if (PHPUNIT)
-        message(STATUS "Using phpunit binary ${PHPUNIT}")
-        add_test(NAME php
-                 COMMAND ${PHPUNIT} ./
-                 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/php)
-    else()
-        message(WARNING "phpunit not found. PHP unit tests disabled." )
-    endif()
-
-    if (PHPCS)
-        message(STATUS "Using phpcs binary ${PHPCS}")
-        add_test(NAME phpcs
-                 COMMAND ${PHPCS} --report-width=120 --colors lib website utils
-                 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-    else()
-        message(WARNING "phpcs not found. PHP linting tests disabled." )
-    endif()
-
-    if (PYLINT)
-        message(STATUS "Using pylint binary ${PYLINT}")
-        add_test(NAME pylint
-                 COMMAND ${PYLINT} nominatim
-                 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-    else()
-        message(WARNING "pylint not found. Python linting tests disabled.")
-    endif()
-
-    if (PYTEST)
-        message(STATUS "Using pytest binary ${PYTEST}")
-        add_test(NAME pytest
-                 COMMAND ${PYTEST} test/python
-                 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-    else()
-        message(WARNING "pytest not found. Python tests disabled." )
-    endif()
+    add_test(NAME phpcs
+             COMMAND phpcs --report-width=120 --colors lib website utils
+             WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
 endif()
 
 #-----------------------------------------------------------------------------
@@ -193,71 +200,3 @@ endif()
 if (BUILD_DOCS)
    add_subdirectory(docs)
 endif()
-
-#-----------------------------------------------------------------------------
-# Manual page
-#-----------------------------------------------------------------------------
-
-if (BUILD_MANPAGE)
-   add_subdirectory(manual)
-endif()
-
-#-----------------------------------------------------------------------------
-# Installation
-#-----------------------------------------------------------------------------
-
-
-include(GNUInstallDirs)
-set(NOMINATIM_DATADIR ${CMAKE_INSTALL_FULL_DATADIR}/${PROJECT_NAME})
-set(NOMINATIM_LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME})
-set(NOMINATIM_CONFIGDIR ${CMAKE_INSTALL_FULL_SYSCONFDIR}/${PROJECT_NAME})
-
-if (BUILD_IMPORTER)
-    configure_file(${PROJECT_SOURCE_DIR}/cmake/tool-installed.tmpl installed.bin)
-    install(PROGRAMS ${PROJECT_BINARY_DIR}/installed.bin
-            DESTINATION ${CMAKE_INSTALL_BINDIR}
-            RENAME nominatim)
-
-    install(DIRECTORY nominatim
-            DESTINATION ${NOMINATIM_LIBDIR}/lib-python
-            FILES_MATCHING PATTERN "*.py"
-            PATTERN __pycache__ EXCLUDE)
-    install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR})
-
-    install(FILES data/country_name.sql
-                  ${COUNTRY_GRID_FILE}
-                  data/words.sql
-            DESTINATION ${NOMINATIM_DATADIR})
-endif()
-
-if (BUILD_OSM2PGSQL)
-    if (${CMAKE_VERSION} VERSION_LESS 3.13)
-        # Installation of subdirectory targets was only introduced in 3.13.
-        # So just copy the osm2pgsql file for older versions.
-        install(PROGRAMS ${PROJECT_BINARY_DIR}/osm2pgsql/osm2pgsql
-                DESTINATION ${NOMINATIM_LIBDIR})
-    else()
-        install(TARGETS osm2pgsql RUNTIME DESTINATION ${NOMINATIM_LIBDIR})
-    endif()
-endif()
-
-if (BUILD_MODULE)
-    install(PROGRAMS ${PROJECT_BINARY_DIR}/module/nominatim.so
-            DESTINATION ${NOMINATIM_LIBDIR}/module)
-endif()
-
-if (BUILD_API)
-    install(DIRECTORY lib-php DESTINATION ${NOMINATIM_LIBDIR})
-endif()
-
-install(FILES settings/env.defaults
-              settings/address-levels.json
-              settings/phrase-settings.json
-              settings/import-admin.style
-              settings/import-street.style
-              settings/import-address.style
-              settings/import-full.style
-              settings/import-extratags.style
-              settings/legacy_icu_tokenizer.yaml
-              settings/icu-rules/extended-unicode-to-asccii.yaml
-        DESTINATION ${NOMINATIM_CONFIGDIR})

CONTRIBUTING.md

@@ -49,18 +49,22 @@ are in process of consolidating the style. The following rules apply:
  * for PHP variables use CamelCase with a prefixing letter indicating the type
    (i - integer, f - float, a - array, s - string, o - object)
 
-The coding style is enforced with PHPCS and pylint. It can be tested with:
+The coding style is enforced with PHPCS and can be tested with:
 
 ```
-phpcs --report-width=120 --colors .
-pylint3 --extension-pkg-whitelist=osmium nominatim
+  phpcs --report-width=120 --colors .
 ```
 
 ## Testing
 
-Before submitting a pull request make sure that the tests pass:
+Before submitting a pull request make sure that the following tests pass:
 
 ```
-  cd build
-  make test
+  cd test/bdd
+  behave -DBUILDDIR=<builddir> db osm2pgsql
+```
+
+```
+  cd test/php
+  phpunit ./
 ```

ChangeLog

@@ -1,26 +1,3 @@
-3.7.0
-
- * switch to dotenv for configuration file
- * introduce 'make install' (reorganising most of the code)
- * introduce nominatim tool as replacement for various php scripts
- * introduce project directories and allow multiple installations from same build
- * clean up BDD tests: drop nose, reorganise step code
- * simplify test database for API BDD tests and autoinstall database
- * port most of the code for command-line tools to Python
-   (thanks to @darkshredder and @AntoJvlt)
- * add tests for all tooling
- * replace pyosmium-get-changes with custom internal implementation using
-   pyosmium
- * improve search for queries with housenumber and partial terms
- * add database versioning
- * use jinja2 for preprocessing SQL files
- * introduce automatic migrations
- * reverse fix preference of interpolations over housenumbers
- * parallelize indexing of postcodes
- * add non-key indexes to speed up housenumber + street searches
- * switch housenumber field in placex to save transliterated names
-
-
 3.6.0
 
  * add full support for searching by and displaying of addr:* tags

README.md

@@ -1,5 +1,4 @@
 [![Build Status](https://github.com/osm-search/Nominatim/workflows/CI%20Tests/badge.svg)](https://github.com/osm-search/Nominatim/actions?query=workflow%3A%22CI+Tests%22)
-[![codecov](https://codecov.io/gh/osm-search/Nominatim/branch/master/graph/badge.svg?token=8P1LXrhCMy)](https://codecov.io/gh/osm-search/Nominatim)
 
 Nominatim
 =========
@@ -25,14 +24,15 @@ Installing and running Nominatim is something for experienced system
 administrators only who can do some trouble-shooting themselves. We are sorry,
 but we can not provide installation support. We are all doing this in our free
 time and there is just so much of that time to go around. Do not open issues in
-our bug tracker if you need help. Use the discussions forum
-or ask for help on [help.openstreetmap.org](https://help.openstreetmap.org/).**
+our bug tracker if you need help. You can ask questions on the mailing list
+(see below) or on [help.openstreetmap.org](https://help.openstreetmap.org/).**
 
 The latest stable release can be downloaded from https://nominatim.org.
 There you can also find [installation instructions for the release](https://nominatim.org/release-docs/latest/admin/Installation), as well as an extensive [Troubleshooting/FAQ section](https://nominatim.org/release-docs/latest/admin/Faq/).
 
-[Detailed installation instructions for current master](https://nominatim.org/release-docs/develop/admin/Installation)
-can be found at nominatim.org as well.
+Detailed installation instructions for the development version can be
+found at [nominatim.org](https://nominatim.org/release-docs/develop/admin/Installation)
+as well.
 
 A quick summary of the necessary steps:
 
@@ -42,15 +42,12 @@ A quick summary of the necessary steps:
         cd build
         cmake ..
         make
-        sudo make install
 
-2. Create a project directory, get OSM data and import:
+2. Get OSM data and import:
 
-        mkdir nominatim-project
-        cd nominatim-project
-        nominatim import --osm-file <your planet file>
+        ./build/utils/setup.php --osm-file <your planet file> --all
 
-3. Point your webserver to the nominatim-project/website directory.
+3. Point your webserver to the ./build/website directory.
 
 
 License
@@ -62,14 +59,13 @@ The source code is available under a GPLv2 license.
 Contributing
 ============
 
-Contributions, bugreport and pull requests are welcome.
-For details see [contribution guide](CONTRIBUTING.md).
+Contributions are welcome. For details see [contribution guide](CONTRIBUTING.md).
+
+Both bug reports and pull requests are welcome.
 
 
-Questions and help
-==================
+Mailing list
+============
 
-For questions, community help and discussions you can use the
-[Github discussions forum](https://github.com/osm-search/Nominatim/discussions)
-or join the
-[geocoding mailing list](https://lists.openstreetmap.org/listinfo/geocoding).
+For questions you can join the geocoding mailing list, see
+https://lists.openstreetmap.org/listinfo/geocoding

SECURITY.md

@@ -1,39 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-All Nominatim releases receive security updates for two years.
-
-The following table lists the end of support for all currently supported
-versions.
-
-| Version | End of support for security updates |
-| ------- | ----------------------------------- |
-| 3.7.x   | 2023-04-05                          |
-| 3.6.x   | 2022-12-12                          |
-| 3.5.x   | 2022-06-05                          |
-| 3.4.x   | 2021-10-24                          |
-
-## Reporting a Vulnerability
-
-If you believe, you have found an issue in Nominatim that has implications on
-security, please send a description of the issue to **security@nominatim.org**.
-You will receive an acknowledgement of your mail within 3 work days where we
-also notify you of the next steps.
-
-## How we Disclose Security Issues
-
-** The following section only applies to security issues found in released
-versions. Issues that concern the master development branch only will be
-fixed immediately on the branch with the corresponding PR containing the
-description of the nature and severity of the issue. **
-
-Patches for identified security issues are applied to all affected versions and
-new minor versions are released. At the same time we release a statement at
-the [Nominatim blog](https://nominatim.org/blog/) describing the nature of the
-incident. Announcements will also be published at the
-[geocoding mailinglist](https://lists.openstreetmap.org/listinfo/geocoding).
-
-## List of Previous Incidents
-
-* 2020-05-04 - [SQL injection issue on /details endpoint](https://lists.openstreetmap.org/pipermail/geocoding/2020-May/002012.html)

cmake/script.tmpl

@@ -0,0 +1,4 @@
+#!@PHP_BIN@ -Cq
+<?php
+require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
+require_once(CONST_BasePath.'/@script_source@');

cmake/tool-installed.tmpl

@@ -1,17 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import os
-
-sys.path.insert(1, '@NOMINATIM_LIBDIR@/lib-python')
-
-os.environ['NOMINATIM_NOMINATIM_TOOL'] = os.path.abspath(__file__)
-
-from nominatim import cli
-
-exit(cli.nominatim(module_dir='@NOMINATIM_LIBDIR@/module',
-                   osm2pgsql_path='@NOMINATIM_LIBDIR@/osm2pgsql',
-                   phplib_dir='@NOMINATIM_LIBDIR@/lib-php',
-                   sqllib_dir='@NOMINATIM_LIBDIR@/lib-sql',
-                   data_dir='@NOMINATIM_DATADIR@',
-                   config_dir='@NOMINATIM_CONFIGDIR@',
-                   phpcgi_path='@PHPCGI_BIN@'))

cmake/tool.tmpl

@@ -1,17 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import os
-
-sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
-
-os.environ['NOMINATIM_NOMINATIM_TOOL'] = os.path.abspath(__file__)
-
-from nominatim import cli
-
-exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',
-                   osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql',
-                   phplib_dir='@CMAKE_SOURCE_DIR@/lib-php',
-                   sqllib_dir='@CMAKE_SOURCE_DIR@/lib-sql',
-                   data_dir='@CMAKE_SOURCE_DIR@/data',
-                   config_dir='@CMAKE_SOURCE_DIR@/settings',
-                   phpcgi_path='@PHPCGI_BIN@'))

cmake/website.tmpl

@@ -0,0 +1,5 @@
+<?php
+@define('CONST_Debug', (isset($_GET['debug']) && $_GET['debug']));
+require_once(dirname(dirname(__FILE__)).'/settings/settings-frontend.php');
+
+require_once(CONST_BasePath.'/@script_source@');

codecov.yml

@@ -1,14 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  status:
-    project: off
-    patch: off
-
-comment:
-  require_changes: true
-  after_n_builds: 2
-
-fixes:
-  - "Nominatim/::"

data/gb_postcode_table.sql

@@ -0,0 +1,26 @@
+-- This data contains Ordnance Survey data © Crown copyright and database right 2010. 
+-- Code-Point Open contains Royal Mail data © Royal Mail copyright and database right 2010.
+-- OS data may be used under the terms of the OS OpenData licence:
+-- http://www.ordnancesurvey.co.uk/oswebsite/opendata/licence/docs/licence.pdf
+
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+SET escape_string_warning = off;
+
+SET search_path = public, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+CREATE TABLE gb_postcode (
+    id integer,
+    postcode character varying(9),
+    geometry geometry,
+    CONSTRAINT enforce_dims_geometry CHECK ((st_ndims(geometry) = 2)),
+    CONSTRAINT enforce_srid_geometry CHECK ((st_srid(geometry) = 4326))
+);
+

data/us_postcode_table.sql

@@ -0,0 +1,16 @@
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+
+SET search_path = public, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+CREATE TABLE us_postcode (
+    postcode text,
+    x double precision,
+    y double precision
+);

data/words.sql

@@ -29787,7 +29787,7 @@ st	5557484
 
 -- prefill word table
 
-select count(precompute_words(v)) from (select distinct svals(name) as v from place) as w where v is not null;
+select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;
 select count(getorcreate_housenumber_id(make_standard_name(v))) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
 
 -- copy the word frequencies

docs/admin/Advanced-Installations.md

@@ -47,16 +47,15 @@ The file `import_multiple_regions.sh` needs to be edited as per your requirement
     
         BASEURL="https://download.geofabrik.de"
         DOWNCOUNTRYPOSTFIX="-latest.osm.pbf"
-
-### Setting up multiple regions
-
+ 
 !!! tip
-    If your database already exists and you want to add more countries,
-    replace the setting up part
+    If your database already exists and you want to add more countries, replace the setting up part
     `${SETUPFILE} --osm-file ${UPDATEDIR}/tmp/combined.osm.pbf --all 2>&1`
     with `${UPDATEFILE} --import-file ${UPDATEDIR}/tmp/combined.osm.pbf --index --index-instances N 2>&1`
     where N is the numbers of CPUs in your system.
 
+### Setting up multiple regions
+
 Run the following command from your Nominatim directory after configuring the file.
 
     bash ./utils/import_multiple_regions.sh
@@ -155,7 +154,7 @@ Make sure that the PostgreSQL server package is installed on the machine
 the PostgreSQL server itself.
 
 Download and compile Nominatim as per standard instructions. Once done, you find
-the normalization library in `build/module/nominatim.so`. Copy the file to
+the nomrmalization library in `build/module/nominatim.so`. Copy the file to
 the database server at a location where it is readable and executable by the
 PostgreSQL server process.
 
@@ -163,11 +162,11 @@ PostgreSQL server process.
 
 On the client side you now need to configure the import to point to the
 correct location of the library **on the database server**. Add the following
-line to your your `.env` file:
+line to your your `settings/local.php` file:
 
 ```php
-NOMINATIM_DATABASE_MODULE_PATH="<directory on the database server where nominatim.so resides>"
+@define('CONST_Database_Module_Path', '<directory on the database server where nominatim.so resides>');
 ```
 
-Now change the `NOMINATIM_DATABASE_DSN` to point to your remote server and continue
+Now change the `CONST_Database_DSN` to point to your remote server and continue
 to follow the [standard instructions for importing](/admin/Import).

docs/admin/Customization.md

@@ -1,101 +0,0 @@
-# Customization of the Database
-
-This section explains in detail how to configure a Nominatim import and
-the various means to use external data.
-
-## External postcode data
-
-Nominatim creates a table of known postcode centroids during import. This table
-is used for searches of postcodes and for adding postcodes to places where the
-OSM data does not provide one. These postcode centroids are mainly computed
-from the OSM data itself. In addition, Nominatim supports reading postcode
-information from an external CSV file, to supplement the postcodes that are
-missing in OSM.
-
-To enable external postcode support, simply put one CSV file per country into
-your project directory and name it `<CC>_postcodes.csv`. `<CC>` must be the
-two-letter country code for which to apply the file. The file may also be
-gzipped. Then it must be called `<CC>_postcodes.csv.gz`.
-
-The CSV file must use commas as a delimiter and have a header line. Nominatim
-expects three columns to be present: `postcode`, `lat` and `lon`. All other
-columns are ignored. `lon` and `lat` must describe the x and y coordinates of the
-postcode centroids in WGS84.
-
-The postcode files are loaded only when there is data for the given country
-in your database. For example, if there is a `us_postcodes.csv` file in your
-project directory but you import only an excerpt of Italy, then the US postcodes
-will simply be ignored.
-
-As a rule, the external postcode data should be put into the project directory
-**before** starting the initial import. Still, you can add, remove and update the
-external postcode data at any time. Simply
-run:
-
-```
-nominatim refresh --postcodes
-```
-
-to make the changes visible in your database. Be aware, however, that the changes
-only have an immediate effect on searches for postcodes. Postcodes that were
-added to places are only updated, when they are reindexed. That usually happens
-only during replication updates.
-
-## Installing Tiger housenumber data for the US
-
-Nominatim is able to use the official [TIGER](https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html)
-address set to complement the OSM house number data in the US. You can add
-TIGER data to your own Nominatim instance by following these steps. The
-entire US adds about 10GB to your database.
-
-  1. Get preprocessed TIGER 2020 data:
-
-        cd $PROJECT_DIR
-        wget https://nominatim.org/data/tiger2020-nominatim-preprocessed.csv.tar.gz
-
-  2. Import the data into your Nominatim database:
-
-        nominatim add-data --tiger-data tiger2020-nominatim-preprocessed.csv.tar.gz
-
-  3. Enable use of the Tiger data in your `.env` by adding:
-
-        echo NOMINATIM_USE_US_TIGER_DATA=yes >> .env
-
-  4. Apply the new settings:
-
-        nominatim refresh --functions
-
-
-See the [developer's guide](../develop/data-sources.md#us-census-tiger) for more
-information on how the data got preprocessed.
-
-## Special phrases import
-
-As described in the [Importation chapter](Import.md), it is possible to
-import special phrases from the wiki with the following command:
-
-```sh
-nominatim special-phrases --import-from-wiki
-```
-
-But, it is also possible to import some phrases from a csv file. 
-To do so, you have access to the following command:
-
-```sh
-nominatim special-phrases --import-from-csv <csv file>
-```
-
-Note that the two previous import commands will update the phrases from your database.
-This means that if you import some phrases from a csv file, only the phrases
-present in the csv file will be kept into the database. All other phrases will
-be removed.
-
-If you want to only add new phrases and not update the other ones you can add
-the argument `--no-replace` to the import command. For example:
-
-```sh
-nominatim special-phrases --import-from-csv <csv file> --no-replace
-```
-
-This will add the phrases present in the csv file into the database without
-removing the other ones.

docs/admin/Deployment.md

@@ -1,7 +1,7 @@
 # Deploying Nominatim
 
 The Nominatim API is implemented as a PHP application. The `website/` directory
-in the project directory contains the configured website. You can serve this
+in the build directory contains the configured website. You can serve this
 in a production environment with any web server that is capable to run
 PHP scripts.
 
@@ -13,11 +13,10 @@ to run a web service. Please refer to the documentation of
 for background information on configuring the services.
 
 !!! Note
-    Throughout this page, we assume that your Nominatim project directory is
-    located in `/srv/nominatim-project` and that you have installed Nominatim
-    using the default installation prefix `/usr/local`. If you have put it
-    somewhere else, you need to adjust the commands and configuration
-    accordingly.
+    Throughout this page, we assume that your Nominatim build directory is
+    located in `/srv/nominatim/build` and the source code in
+    `/srv/nominatim/Nominatim`. If you have put it somewhere else, you
+    need to adjust the commands and configuration accordingly.
 
     We further assume that your web server runs as user `www-data`. Older
     versions of CentOS may still use the user name `apache`. You also need
@@ -30,7 +29,7 @@ web server user. You can check that the permissions are correct by accessing
 on of the php files as the web server user:
 
 ``` sh
-sudo -u www-data head -n 1 /srv/nominatim-project/website/search.php
+sudo -u www-data head -n 1 /srv/nominatim/build/website/search.php
 ```
 
 If this shows a permission error, then you need to adapt the permissions of
@@ -41,11 +40,11 @@ web server access. At a minimum the following SELinux labelling should be done
 for Nominatim:
 
 ``` sh
-sudo semanage fcontext -a -t httpd_sys_content_t "/usr/local/nominatim/lib/lib-php(/.*)?"
-sudo semanage fcontext -a -t httpd_sys_content_t "/srv/nominatim-project/website(/.*)?"
-sudo semanage fcontext -a -t lib_t "/srv/nominatim-project/module/nominatim.so"
-sudo restorecon -R -v /usr/local/lib/nominatim
-sudo restorecon -R -v /srv/nominatim-project
+sudo semanage fcontext -a -t httpd_sys_content_t "/srv/nominatim/Nominatim/(website|lib|settings)(/.*)?"
+sudo semanage fcontext -a -t httpd_sys_content_t "/srv/nominatim/build/(website|settings)(/.*)?"
+sudo semanage fcontext -a -t lib_t "/srv/nominatim/build/module/nominatim.so"
+sudo restorecon -R -v /srv/nominatim/Nominatim
+sudo restorecon -R -v /srv/nominatim/build
 ```
 
 ## Nominatim with Apache
@@ -66,13 +65,13 @@ Make sure your Apache configuration contains the required permissions for the
 directory and create an alias:
 
 ``` apache
-<Directory "/srv/nominatim-project/website">
+<Directory "/srv/nominatim/build/website">
   Options FollowSymLinks MultiViews
   AddType text/html   .php
   DirectoryIndex search.php
   Require all granted
 </Directory>
-Alias /nominatim /srv/nominatim-project/website
+Alias /nominatim /srv/nominatim/build/website
 ```
 
 After making changes in the apache config you need to restart apache.
@@ -111,7 +110,7 @@ Tell nginx that php files are special and to fastcgi_pass to the php-fpm
 unix socket by adding the location definition to the default configuration.
 
 ``` nginx
-root /srv/nominatim-project/website;
+root /srv/nominatim/build/website;
 index search.php;
 location / {
     try_files $uri $uri/ @php;

docs/admin/Faq.md

@@ -16,7 +16,7 @@ was killed. If it looks like this:
 then you can resume with the following command:
 
 ```sh
-nominatim import --continue indexing
+./utils/setup.php --index --create-search-indices --create-country-names
 ```
 
 If the reported rank is 26 or higher, you can also safely add `--index-noanalyse`.
@@ -31,7 +31,7 @@ list for hints.
 If it happened during index creation you can try rerunning the step with
 
 ```sh
-nominatim import --continue indexing
+./utils/setup.php --create-search-indices --ignore-errors
 ```
 
 Otherwise it's best to start the full setup from the beginning.
@@ -93,7 +93,7 @@ on a non-managed machine.
 
 ### I see the error: "function transliteration(text) does not exist"
 
-Reinstall the nominatim functions with `nominatim refresh --functions`
+Reinstall the nominatim functions with `setup.php --create--functions`
 and check for any errors, e.g. a missing `nominatim.so` file.
 
 ### I see the error: "ERROR: mmap (remap) failed"
@@ -113,8 +113,7 @@ Double-check clang is installed. Instead of `make` try running `make CLANG=true`
 
 ### nominatim UPDATE failed: ERROR: buffer 179261 is not owned by resource owner Portal
 
-Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168)
-during the initial import of the database. It's
+Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168) during the initial import of the database. It's
 something PostgreSQL internal Nominatim doesn't control. And PostgreSQL forums
 suggest it's threading related but definitely some kind of crash of a process.
 Users reported either rebooting the server, different hardware or just trying
@@ -203,7 +202,7 @@ See the installation instructions for a full list of required packages.
 ### I forgot to delete the flatnodes file before starting an import.
 
 That's fine. For each import the flatnodes file get overwritten.
-See [https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage](https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage)
+See [https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage]()
 for more information.
 
 
@@ -212,3 +211,11 @@ for more information.
 ### Can I import negative OSM ids into Nominatim?
 
 See [this question of Stackoverflow](https://help.openstreetmap.org/questions/64662/nominatim-flatnode-with-negative-id).
+
+### Missing XML or text declaration
+
+The website might show: `XML Parsing Error: XML or text declaration not at start of entity Location.`
+
+Make sure there are no spaces at the beginning of your `settings/local.php` file.
+
+

docs/admin/Import.md

@@ -1,54 +1,22 @@
 # Importing the Database
 
 The following instructions explain how to create a Nominatim database
-from an OSM planet file. It is assumed that you have already successfully
-installed the Nominatim software itself and the `nominatim` tool can be found
-in your `PATH`. If this is not the case, return to the
-[installation page](Installation.md).
+from an OSM planet file and how to keep the database up to date. It
+is assumed that you have already successfully installed the Nominatim
+software itself, if not return to the [installation page](Installation.md).
 
-## Creating the project directory
+## Configuration setup in settings/local.php
 
-Before you start the import, you should create a project directory for your
-new database installation. This directory receives all data that is related
-to a single Nominatim setup: configuration, extra data, etc. Create a project
-directory apart from the Nominatim software and change into the directory:
+The Nominatim server can be customized via the file `settings/local.php`
+in the build directory. Note that this is a PHP file, so it must always
+start like this:
 
-```
-mkdir ~/nominatim-planet
-cd ~/nominatim-planet
-```
+    <?php
 
-In the following, we refer to the project directory as `$PROJECT_DIR`. To be
-able to copy&paste instructions, you can export the appropriate variable:
-
-```
-export PROJECT_DIR=~/nominatim-planet
-```
-
-The Nominatim tool assumes per default that the current working directory is
-the project directory but you may explicitly state a different directory using
-the `--project-dir` parameter. The following instructions assume that you run
-all commands from the project directory.
-
-!!! tip "Migration Tip"
-
-    Nominatim used to be run directly from the build directory until version 3.6.
-    Essentially, the build directory functioned as the project directory
-    for the database installation. This setup still works and can be useful for
-    development purposes. It is not recommended anymore for production setups.
-    Create a project directory that is separate from the Nominatim software.
-
-### Configuration setup in `.env`
-
-The Nominatim server can be customized via an `.env` configuration file in the 
-project directory. This is a file in [dotenv](https://github.com/theskumar/python-dotenv)
-format which looks the same as variable settings in a standard shell environment.
-You can also set the same configuration via environment variables. All
-settings have a `NOMINATIM_` prefix to avoid conflicts with other environment
-variables.
+without any leading spaces.
 
 There are lots of configuration settings you can tweak. Have a look
-at `Nominatim/settings/env.default` for a full list. Most should have a sensible default.
+at `settings/default.php` for a full list. Most should have a sensible default.
 
 #### Flatnode files
 
@@ -56,9 +24,9 @@ If you plan to import a large dataset (e.g. Europe, North America, planet),
 you should also enable flatnode storage of node locations. With this
 setting enabled, node coordinates are stored in a simple file instead
 of the database. This will save you import time and disk storage.
-Add to your `.env`:
+Add to your `settings/local.php`:
 
-    NOMINATIM_FLATNODE_FILE="/path/to/flatnode.file"
+    @define('CONST_Osm2pgsql_Flatnode_File', '/path/to/flatnode.file');
 
 Replace the second part with a suitable path on your system and make sure
 the directory exists. There should be at least 75GB of free space.
@@ -70,9 +38,9 @@ the directory exists. There should be at least 75GB of free space.
 Wikipedia can be used as an optional auxiliary data source to help indicate
 the importance of OSM features. Nominatim will work without this information
 but it will improve the quality of the results if this is installed.
-This data is available as a binary download. Put it into your project directory:
+This data is available as a binary download:
 
-    cd $PROJECT_DIR
+    cd $NOMINATIM_SOURCE_DIR/data
     wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
 
 The file is about 400MB and adds around 4GB to the Nominatim database.
@@ -80,22 +48,17 @@ The file is about 400MB and adds around 4GB to the Nominatim database.
 !!! tip
     If you forgot to download the wikipedia rankings, you can also add
     importances after the import. Download the files, then run
-    `nominatim refresh --wiki-data --importance`. Updating importances for
-    a planet can take a couple of hours.
+    `./utils/setup.php --import-wikipedia-articles`
+    and `./utils/update.php --recompute-importance`.
 
-### External postcodes
+### Great Britain, USA postcodes
 
-Nominatim can use postcodes from an external source to improve searching with
-postcodes. We provide precomputed postcodes sets for the US (using TIGER data)
-and the UK (using the [CodePoint OpenData set](https://osdatahub.os.uk/downloads/open/CodePointOpen).
-This data can be optionally downloaded into the project directory:
+Nominatim can use postcodes from an external source to improve searches that
+involve a GB or US postcode. This data can be optionally downloaded:
 
-    cd $PROJECT_DIR
-    wget https://www.nominatim.org/data/gb_postcodes.csv.gz
-    wget https://www.nominatim.org/data/us_postcodes.csv.gz
-
-You can also add your own custom postcode sources, see
-[Customization of postcodes](Customization.md#external-postcode-data).
+    cd $NOMINATIM_SOURCE_DIR/data
+    wget https://www.nominatim.org/data/gb_postcode_data.sql.gz
+    wget https://www.nominatim.org/data/us_postcode_data.sql.gz
 
 ## Choosing the data to import
 
@@ -123,14 +86,11 @@ that Nominatim cannot compute the areas for some administrative areas.
 About half of the data in Nominatim's database is not really used for serving
 the API. It is only there to allow the data to be updated from the latest
 changes from OSM. For many uses these dynamic updates are not really required.
-If you don't plan to apply updates, you can run the import with the
-`--no-updates` parameter. This will drop the dynamic part of the database as
-soon as it is not required anymore.
-
-You can also drop the dynamic part later using the following command:
+If you don't plan to apply updates, the dynamic part of the database can be
+safely dropped using the following command:
 
 ```
-nominatim freeze
+./utils/setup.php --drop
 ```
 
 Note that you still need to provide for sufficient disk space for the initial
@@ -164,7 +124,7 @@ import styles available which only read selected data:
   Like the full style but also adds most of the OSM tags into the extratags
   column.
 
-The style can be changed with the configuration `NOMINATIM_IMPORT_STYLE`.
+The style can be changed with the configuration `CONST_Import_Style`.
 
 To give you an idea of the impact of using the different styles, the table
 below gives rough estimates of the final database size after import of a
@@ -193,15 +153,12 @@ can be found in the development section.
     [Geofabrik](https://download.geofabrik.de).
 
 Download the data to import. Then issue the following command
-from the **project directory** to start the import:
+from the **build directory** to start the import:
 
 ```sh
-nominatim import --osm-file <data file> 2>&1 | tee setup.log
+./utils/setup.php --osm-file <data file> --all 2>&1 | tee setup.log
 ```
 
-The **project directory** is the one that you have set up at the beginning.
-See [creating the project directory](Import#creating-the-project-directory).
-
 ### Notes on full planet imports
 
 Even on a perfectly configured machine
@@ -235,19 +192,29 @@ MB. Make sure you leave enough RAM for PostgreSQL and osm2pgsql as mentioned
 above. If the system starts swapping or you are getting out-of-memory errors,
 reduce the cache size or even consider using a flatnode file.
 
-
-### Testing the installation
+### Verify the import
 
 Run this script to verify all required tables and indices got created successfully.
 
 ```sh
-nominatim admin --check-database
+./utils/check_import_finished.php
 ```
 
+### Setting up the website
+
+Run the following command to set up the configuration file for the API frontend
+`settings/settings-frontend.php`. These settings are used in website/*.php files.
+
+```sh
+./utils/setup.php --setup-website
+```
+!!! Note
+    This step is not necessary if you use `--all` option while setting up the DB.
+
 Now you can try out your installation by running:
 
 ```sh
-nominatim serve
+make serve
 ```
 
 This runs a small test server normally used for development. You can use it
@@ -255,9 +222,6 @@ to verify that your installation is working. Go to
 `http://localhost:8088/status.php` and you should see the message `OK`.
 You can also run a search query, e.g. `http://localhost:8088/search.php?q=Berlin`.
 
-Note that search query is not supported for reverse-only imports. You can run a
-reverse query, e.g. `http://localhost:8088/reverse.php?lat=27.1750090510034&lon=78.04209025`.
-
 To run Nominatim via webservers like Apache or nginx, please read the
 [Deployment chapter](Deployment.md).
 
@@ -268,7 +232,7 @@ planner to make the right decisions. Recomputing them can improve the performanc
 of forward geocoding in particular under high load. To recompute word counts run:
 
 ```sh
-nominatim refresh --word-counts
+./utils/update.php --recompute-word-counts
 ```
 
 This will take a couple of hours for a full planet installation. You can
@@ -278,14 +242,44 @@ running this function.
 
 If you want to be able to search for places by their type through
 [special key phrases](https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases)
-you also need to import these key phrases like this:
+you also need to enable these key phrases like this:
 
-```sh
-nominatim special-phrases --import-from-wiki
-```
+    ./utils/specialphrases.php --wiki-import > specialphrases.sql
+    psql -d nominatim -f specialphrases.sql
 
 Note that this command downloads the phrases from the wiki link above. You
 need internet access for the step.
 
-You can also import special phrases from a csv file, for more 
-information please read the [Customization chapter](Customization.md).
+
+## Installing Tiger housenumber data for the US
+
+Nominatim is able to use the official [TIGER](https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html)
+address set to complement the OSM house number data in the US. You can add
+TIGER data to your own Nominatim instance by following these steps. The
+entire US adds about 10GB to your database.
+
+  1. Get preprocessed TIGER 2019 data and unpack it into the
+     data directory in your Nominatim sources:
+
+        cd Nominatim/data
+        wget https://nominatim.org/data/tiger2019-nominatim-preprocessed.tar.gz
+        tar xf tiger2019-nominatim-preprocessed.tar.gz
+
+  2. Import the data into your Nominatim database:
+
+        ./utils/setup.php --import-tiger-data
+
+  3. Enable use of the Tiger data in your `settings/local.php` by adding:
+
+         @define('CONST_Use_US_Tiger_Data', true);
+
+  4. Apply the new settings:
+
+```sh
+    ./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
+```
+
+
+See the [developer's guide](../develop/data-sources.md#us-census-tiger) for more
+information on how the data got preprocessed.
+

docs/admin/Installation.md

@@ -17,7 +17,6 @@ and can't offer support.
 
   * [Docker](https://github.com/mediagis/nominatim-docker)
   * [Docker on Kubernetes](https://github.com/peter-evans/nominatim-k8s)
-  * [Kubernetes with Helm](https://github.com/robjuz/helm-charts/blob/master/charts/nominatim/README.md)
   * [Ansible](https://github.com/synthesio/infra-ansible-nominatim)
 
 ## Prerequisites
@@ -31,31 +30,23 @@ For compiling:
   * [proj](https://proj.org/)
   * [bzip2](http://www.bzip.org/)
   * [zlib](https://www.zlib.net/)
-  * [ICU](http://site.icu-project.org/)
   * [Boost libraries](https://www.boost.org/), including system and filesystem
   * PostgreSQL client libraries
   * a recent C++ compiler (gcc 5+ or Clang 3.8+)
 
 For running Nominatim:
 
-  * [PostgreSQL](https://www.postgresql.org) (9.5+ will work, 11+ strongly recommended)
+  * [PostgreSQL](https://www.postgresql.org) (9.3+)
   * [PostGIS](https://postgis.net) (2.2+)
-  * [Python 3](https://www.python.org/) (3.6+)
-  * [Psycopg2](https://www.psycopg.org) (2.7+)
-  * [Python Dotenv](https://github.com/theskumar/python-dotenv)
-  * [psutil](https://github.com/giampaolo/psutil)
-  * [Jinja2](https://palletsprojects.com/p/jinja/)
-  * [PyICU](https://pypi.org/project/PyICU/)
-  * [PyYaml](https://pyyaml.org/) (5.1+)
-  * [datrie](https://github.com/pytries/datrie)
+  * [Python 3](https://www.python.org/)
+  * [Psycopg2](https://www.psycopg.org)
   * [PHP](https://php.net) (7.0 or later)
   * PHP-pgsql
   * PHP-intl (bundled with PHP)
-  * PHP-cgi (for running queries from the command line)
 
 For running continuous updates:
 
-  * [pyosmium](https://osmcode.org/pyosmium/)
+  * [pyosmium](https://osmcode.org/pyosmium/) (with Python 3)
 
 For dependencies for running tests and building documentation, see
 the [Development section](../develop/Development-Environment.md).
@@ -151,16 +142,6 @@ build at the same level as the Nominatim source directory run:
 ```
 cmake ../Nominatim
 make
-sudo make install
-```
-
-Nominatim installs itself into `/usr/local` per default. To choose a different
-installation directory add `-DCMAKE_INSTALL_PREFIX=<install root>` to the
-cmake command. Make sure that the `bin` directory is available in your path
-in that case, e.g.
-
-```
-export PATH=<install root>/bin:$PATH
 ```
 
 Now continue with [importing the database](Import.md).

docs/admin/Migration.md

@@ -1,66 +1,10 @@
 # Database Migrations
 
-Since version 3.7.0 Nominatim offers automatic migrations. Please follow
-the following steps:
+This page describes database migrations necessary to update existing databases
+to newer versions of Nominatim.
 
-* stop any updates that are potentially running
-* update Nominatim to the newer version
-* go to your project directory and run `nominatim admin --migrate`
-* (optionally) restart updates
-
-Below you find additional migrations and hints about other structural and
-breaking changes. **Please read them before running the migration.**
-
-!!! note
-    If you are migrating from a version <3.6, then you still have to follow
-    the manual migration steps up to 3.6.
-
-## 3.6.0 -> 3.7.0
-
-### New format and name of configuration file
-
-The configuration for an import is now saved in a `.env` file in the project
-directory. This file follows the dotenv format. For more information, see
-the [installation chapter](Import.md#configuration-setup-in-env).
-
-To migrate to the new system, create a new project directory, add the `.env`
-file and port your custom configuration from `settings/local.php`. Most
-settings are named similar and only have received a `NOMINATIM_` prefix.
-Use the default settings in `settings/env.defaults` as a reference.
-
-### New location for data files
-
-External data files for Wikipedia importance, postcodes etc. are no longer
-expected to reside in the source tree by default. Instead they will be searched
-in the project directory. If you have an automated setup script you must
-either adapt the download location or explicitly set the location of the
-files to the old place in your `.env`.
-
-### Introducing `nominatim` command line tool
-
-The various php utilities have been replaced with a single `nominatim`
-command line tool. Make sure to adapt any scripts. There is no direct 1:1
-matching between the old utilities and the commands of nominatim CLI. The
-following list gives you a list of nominatim sub-commands that contain
-functionality of each script:
-
-* ./utils/setup.php: `import`, `freeze`, `refresh`
-* ./utils/update.php: `replication`, `add-data`, `index`, `refresh`
-* ./utils/specialphrases.php: `special-phrases`
-* ./utils/check_import_finished.php: `admin`
-* ./utils/warm.php: `admin`
-* ./utils/export.php: `export`
-
-Try `nominatim <command> --help` for more information about each subcommand.
-
-`./utils/query.php` no longer exists in its old form. `nominatim search`
-provides a replacement but returns different output.
-
-### Switch to normalized house numbers
-
-The housenumber column in the placex table uses now normalized version.
-The automatic migration step will convert the column but this may take a
-very long time. It is advisable to take the machine offline while doing that.
+SQL statements should be executed from the PostgreSQL commandline. Execute
+`psql nominatim` to enter command line mode.
 
 ## 3.5.0 -> 3.6.0
 
@@ -124,14 +68,6 @@ configuration file, run the following command after updating:
 ./utils/setup.php --setup-website
 ```
 
-### Update SQL code
-
-To update the SQL code to the leatest version run:
-
-```
-./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
-```
-
 ## 3.4.0 -> 3.5.0
 
 ### New Wikipedia/Wikidata importance tables

docs/admin/Setup-Nominatim-UI.md

@@ -10,11 +10,12 @@ installation. For more details, please also have a look at the
 
 ## Installing nominatim-ui
 
-We provide regular releases of nominatim-ui that contain the packaged website.
-They do not need any special installation. Just download, configure
-and run it. Grab the latest release from
-[nominatim-ui's Github release page](https://github.com/osm-search/nominatim-ui/releases)
-and unpack it. You can use `nominatim-ui-x.x.x.tar.gz` or `nominatim-ui-x.x.x.zip`.
+nominatim-ui does not need any special installation, just download, configure
+and run it.
+
+Clone the source from github:
+
+    git clone https://github.com/osm-search/nominatim-ui
 
 Copy the example configuration into the right place:
 

docs/admin/Tokenizers.md

@@ -1,205 +0,0 @@
-# Tokenizers
-
-The tokenizer module in Nominatim is responsible for analysing the names given
-to OSM objects and the terms of an incoming query in order to make sure, they
-can be matched appropriately.
-
-Nominatim offers different tokenizer modules, which behave differently and have
-different configuration options. This sections describes the tokenizers and how
-they can be configured.
-
-!!! important
-    The use of a tokenizer is tied to a database installation. You need to choose
-    and configure the tokenizer before starting the initial import. Once the import
-    is done, you cannot switch to another tokenizer anymore. Reconfiguring the
-    chosen tokenizer is very limited as well. See the comments in each tokenizer
-    section.
-
-## Legacy tokenizer
-
-The legacy tokenizer implements the analysis algorithms of older Nominatim
-versions. It uses a special Postgresql module to normalize names and queries.
-This tokenizer is currently the default.
-
-To enable the tokenizer add the following line to your project configuration:
-
-```
-NOMINATIM_TOKENIZER=legacy
-```
-
-The Postgresql module for the tokenizer is available in the `module` directory
-and also installed with the remainder of the software under
-`lib/nominatim/module/nominatim.so`. You can specify a custom location for
-the module with
-
-```
-NOMINATIM_DATABASE_MODULE_PATH=<path to directory where nominatim.so resides>
-```
-
-This is in particular useful when the database runs on a different server.
-See [Advanced installations](Advanced-Installations.md#importing-nominatim-to-an-external-postgresql-database) for details.
-
-There are no other configuration options for the legacy tokenizer. All
-normalization functions are hard-coded.
-
-## ICU tokenizer
-
-!!! danger
-    This tokenizer is currently in active development and still subject
-    to backwards-incompatible changes.
-
-The ICU tokenizer uses the [ICU library](http://site.icu-project.org/) to
-normalize names and queries. It also offers configurable decomposition and
-abbreviation handling.
-
-### How it works
-
-On import the tokenizer processes names in the following four stages:
-
-1. The **Normalization** part removes all non-relevant information from the
-   input.
-2. Incoming names are now converted to **full names**. This process is currently
-   hard coded and mostly serves to handle name tags from OSM that contain
-   multiple names (e.g. [Biel/Bienne](https://www.openstreetmap.org/node/240097197)).
-3. Next the tokenizer creates **variants** from the full names. These variants
-   cover decomposition and abbreviation handling. Variants are saved to the
-   database, so that it is not necessary to create the variants for a search
-   query.
-4. The final **Tokenization** step converts the names to a simple ASCII form,
-   potentially removing further spelling variants for better matching.
-
-At query time only stage 1) and 4) are used. The query is normalized and
-tokenized and the resulting string used for searching in the database.
-
-### Configuration
-
-The ICU tokenizer is configured using a YAML file which can be configured using
-`NOMINATIM_TOKENIZER_CONFIG`. The configuration is read on import and then
-saved as part of the internal database status. Later changes to the variable
-have no effect.
-
-Here is an example configuration file:
-
-``` yaml
-normalization:
-    - ":: lower ()"
-    - "ß > 'ss'" # German szet is unimbigiously equal to double ss
-transliteration:
-    - !include /etc/nominatim/icu-rules/extended-unicode-to-asccii.yaml
-    - ":: Ascii ()"
-variants:
-    - language: de
-      words:
-        - ~haus => haus
-        - ~strasse -> str
-    - language: en
-      words: 
-        - road -> rd
-        - bridge -> bdge,br,brdg,bri,brg
-```
-
-The configuration file contains three sections:
-`normalization`, `transliteration`, `variants`.
-
-The normalization and transliteration sections each must contain a list of
-[ICU transformation rules](https://unicode-org.github.io/icu/userguide/transforms/general/rules.html).
-The rules are applied in the order in which they appear in the file.
-You can also include additional rules from external yaml file using the
-`!include` tag. The included file must contain a valid YAML list of ICU rules
-and may again include other files.
-
-!!! warning
-    The ICU rule syntax contains special characters that conflict with the
-    YAML syntax. You should therefore always enclose the ICU rules in
-    double-quotes.
-
-The variants section defines lists of replacements which create alternative
-spellings of a name. To create the variants, a name is scanned from left to
-right and the longest matching replacement is applied until the end of the
-string is reached.
-
-The variants section must contain a list of replacement groups. Each group
-defines a set of properties that describes where the replacements are
-applicable. In addition, the word section defines the list of replacements
-to be made. The basic replacement description is of the form:
-
-```
-<source>[,<source>[...]] => <target>[,<target>[...]]
-```
-
-The left side contains one or more `source` terms to be replaced. The right side
-lists one or more replacements. Each source is replaced with each replacement
-term.
-
-!!! tip
-    The source and target terms are internally normalized using the
-    normalization rules given in the configuration. This ensures that the
-    strings match as expected. In fact, it is better to use unnormalized
-    words in the configuration because then it is possible to change the
-    rules for normalization later without having to adapt the variant rules.
-
-#### Decomposition
-
-In its standard form, only full words match against the source. There
-is a special notation to match the prefix and suffix of a word:
-
-``` yaml
-- ~strasse => str  # matches "strasse" as full word and in suffix position
-- hinter~ => hntr  # matches "hinter" as full word and in prefix position
-```
-
-There is no facility to match a string in the middle of the word. The suffix
-and prefix notation automatically trigger the decomposition mode: two variants
-are created for each replacement, one with the replacement attached to the word
-and one separate. So in above example, the tokenization of "hauptstrasse" will
-create the variants "hauptstr" and "haupt str". Similarly, the name "rote strasse"
-triggers the variants "rote str" and "rotestr". By having decomposition work
-both ways, it is sufficient to create the variants at index time. The variant
-rules are not applied at query time.
-
-To avoid automatic decomposition, use the '|' notation:
-
-``` yaml
-- ~strasse |=> str
-```
-
-simply changes "hauptstrasse" to "hauptstr" and "rote strasse" to "rote str".
-
-#### Initial and final terms
-
-It is also possible to restrict replacements to the beginning and end of a
-name:
-
-``` yaml
-- ^south => s  # matches only at the beginning of the name
-- road$ => rd  # matches only at the end of the name
-```
-
-So the first example would trigger a replacement for "south 45th street" but
-not for "the south beach restaurant".
-
-#### Replacements vs. variants
-
-The replacement syntax `source => target` works as a pure replacement. It changes
-the name instead of creating a variant. To create an additional version, you'd
-have to write `source => source,target`. As this is a frequent case, there is
-a shortcut notation for it:
-
-```
-<source>[,<source>[...]] -> <target>[,<target>[...]]
-```
-
-The simple arrow causes an additional variant to be added. Note that
-decomposition has an effect here on the source as well. So a rule
-
-``` yaml
-- "~strasse -> str"
-```
-
-means that for a word like `hauptstrasse` four variants are created:
-`hauptstrasse`, `haupt strasse`, `hauptstr` and `haupt str`.
-
-### Reconfiguration
-
-Changing the configuration after the import is currently not possible, although
-this feature may be added at a later time.

docs/admin/Update.md

@@ -1,10 +1,8 @@
 # Updating the Database
 
 There are many different ways to update your Nominatim database.
-The following section describes how to keep it up-to-date using
-an [online replication service for OpenStreetMap data](https://wiki.openstreetmap.org/wiki/Planet.osm/diffs)
-For a list of other methods to add or update data see the output of
-`nominatim add-data --help`.
+The following section describes how to keep it up-to-date with Pyosmium.
+For a list of other methods see the output of `./utils/update.php --help`.
 
 !!! important
     If you have configured a flatnode file for the import, then you
@@ -19,37 +17,50 @@ Run (as the same user who will later run the updates):
 pip3 install --user osmium
 ```
 
+Nominatim needs a tool called `pyosmium-get-changes` which comes with
+Pyosmium. You need to tell Nominatim where to find it. Add the
+following line to your `settings/local.php`:
+
+    @define('CONST_Pyosmium_Binary', '/home/user/.local/bin/pyosmium-get-changes');
+
+The path above is fine if you used the `--user` parameter with pip.
+Replace `user` with your user name.
+
 #### Setting up the update process
 
 Next the update needs to be initialised. By default Nominatim is configured
 to update using the global minutely diffs.
 
 If you want a different update source you will need to add some settings
-to `.env`. For example, to use the daily country extracts
+to `settings/local.php`. For example, to use the daily country extracts
 diffs for Ireland from Geofabrik add the following:
 
-    # base URL of the replication service
-    NOMINATIM_REPLICATION_URL="https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates"
-    # How often upstream publishes diffs (in seconds)
-    NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400
-    # How long to sleep if no update found yet (in seconds)
-    NOMINATIM_REPLICATION_RECHECK_INTERVAL=900
+    // base URL of the replication service
+    @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates');
+    // How often upstream publishes diffs
+    @define('CONST_Replication_Update_Interval', '86400');
+    // How long to sleep if no update found yet
+    @define('CONST_Replication_Recheck_Interval', '900');
 
 To set up the update process now run the following command:
 
-    nominatim replication --init
+    ./utils/update.php --init-updates
 
 It outputs the date where updates will start. Recheck that this date is
 what you expect.
 
-The `replication --init` command needs to be rerun whenever the replication
-service is changed.
+The `--init-updates` command needs to be rerun whenever the replication service
+is changed.
 
 #### Updating Nominatim
 
 The following command will keep your database constantly up to date:
 
-    nominatim replication
+    ./utils/update.php --import-osmosis-all
+
+(Note that even though the old name "import-osmosis-all" has been kept for
+compatibility reasons, Osmosis is not required to run this - it uses pyosmium
+behind the scenes.)
 
 If you have imported multiple country extracts and want to keep them
 up-to-date, [Advanced installations section](Advanced-Installations.md) contains instructions 

docs/api/Faq.md

@@ -58,4 +58,4 @@ The [Overpass API](https://wiki.openstreetmap.org/wiki/Overpass_API) is more
 suited for these kinds of queries.
 
 That said if you installed your own Nominatim instance you can use the
-`nominatim export` PHP script as basis to return such lists.
+`/utils/export.php` PHP script as basis to return such lists.

docs/api/Reverse.md

@@ -162,7 +162,7 @@ This overrides the specified machine readable format. (Default: 0)
   "licence":"Data © OpenStreetMap contributors, ODbL 1.0. https:\/\/www.openstreetmap.org\/copyright",
   "osm_type":"way",
   "osm_id":"280940520",
-  "lat":"-34.4391708",
+"lat":"-34.4391708",
   "lon":"-58.7064573",
   "place_rank":"26",
   "category":"highway",

docs/api/Status.md

@@ -35,16 +35,10 @@ will return HTTP code 200 and a structure
   {
       "status": 0,
       "message": "OK",
-      "data_updated": "2020-05-04T14:47:00+00:00",
-      "software_version": "3.6.0-0",
-      "database_version": "3.6.0-0"
+      "data_updated": "2020-05-04T14:47:00+00:00"
   }
 ```
 
-The `software_version` field contains the version of Nominatim used to serve
-the API. The `database_version` field contains the version of the data format
-in the database.
-
 On error will also return HTTP status code 200 and a structure with error
 code and message, e.g.
 

docs/develop/Development-Environment.md

@@ -26,14 +26,12 @@ following packages should get you started:
 ## Prerequisites for testing and documentation
 
 The Nominatim test suite consists of behavioural tests (using behave) and
-unit tests (using PHPUnit for PHP code and pytest for Python code).
-It has the following additional requirements:
+unit tests (using PHPUnit). It has the following additional requirements:
 
-* [behave test framework](https://behave.readthedocs.io) >= 1.2.6
+* [behave test framework](https://behave.readthedocs.io) >= 1.2.5
+* [nose](https://nose.readthedocs.io)
 * [phpunit](https://phpunit.de) >= 7.3
 * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer)
-* [Pylint](https://pylint.org/) (2.6.0 is used for the CI)
-* [pytest](https://pytest.org)
 
 The documentation is built with mkdocs:
 
@@ -49,9 +47,9 @@ To install all necessary packages run:
 
 ```sh
 sudo apt install php-cgi phpunit php-codesniffer \
-                 python3-pip python3-setuptools python3-dev pylint
+                 python3-pip python3-setuptools python3-dev
 
-pip3 install --user behave mkdocs pytest
+pip3 install --user behave nose mkdocs
 ```
 
 The `mkdocs` executable will be located in `.local/bin`. You may have to add
@@ -80,15 +78,58 @@ echo 'export PATH=~/.config/composer/vendor/bin:$PATH' > ~/.profile
 
 ## Executing Tests
 
-All tests are located in the `/test` directory.
+All tests are located in the `\test` directory.
 
-To run all tests just go to the build directory and run make:
+### Preparing the test database
+
+Some of the behavioural test expect a test database to be present. You need at
+least 2GB RAM and 10GB disk space to create the database.
+
+First create a separate directory for the test DB and fetch the test planet
+data and the Tiger data for South Dakota:
+
+```
+mkdir testdb
+cd testdb
+wget https://www.nominatim.org/data/test/nominatim-api-testdata.pbf
+wget -O - https://nominatim.org/data/tiger2018-nominatim-preprocessed.tar.gz | tar xz --wildcards --no-anchored '46*'
+```
+
+Configure and build Nominatim in the usual way:
+
+```
+cmake $USERNAME/Nominatim
+make
+```
+
+Copy the test settings:
+
+```
+cp $USERNAME/Nominatim/test/testdb/local.php settings/
+```
+
+Inspect the file to check that all settings are correct for your local setup.
+
+Now you can import the test database:
+
+```
+dropdb --if-exists test_api_nominatim
+./utils/setup.php --all --osm-file nominatim-api-testdb.pbf 2>&1 | tee import.log
+./utils/specialphrases.php --wiki-import | psql -d test_api_nominatim 2>&1 | tee -a import.log
+./utils/setup.php --import-tiger-data 2>&1 | tee -a import.log
+```
+
+### Running the tests
+
+To run all tests just go to the test directory and run make:
 
 ```sh
-cd build
-make test
+cd test
+make
 ```
 
+To skip tests that require the test database, run `make no-test-db` instead.
+
 For more information about the structure of the tests and how to change and
 extend the test suite, see the [Testing chapter](Testing.md).
 

docs/develop/Import.md

@@ -29,7 +29,7 @@ once with `class` of `highway` and once with a `class` of `bridge`. Thus the
 ## Configuring the Import
 
 How tags are interpreted and assigned to the different `place` columns can be
-configured via the import style configuration file (`NOMINATIM_IMPORT_STYLE`). This
+configured via the import style configuration file (`CONST_Import_style`). This
 is a JSON file which contains a list of rules which are matched against every
 tag of every object and then assign the tag its specific role.
 

docs/develop/Postcodes.md

@@ -14,7 +14,7 @@ country's format, e.g. if Swiss postcodes are 4 digits.
 ## Regular updating calculated postcodes
 
 The script to rerun the calculation is
-`nominatim refresh --postcodes`
+`build/utils/update.php --calculate-postcodes`
 and runs once per night on nominatim.openstreetmap.org.
 
 

docs/develop/Ranking.md

@@ -87,9 +87,9 @@ into the database. There are a few hard-coded rules for the assignment:
     * highway nodes
     * landuse that is not an area
 
-Other than that, the ranks can be freely assigned via the JSON file according
-to their type and the country they are in. The name of the config file to be
-used can be changed with the setting `NOMINATIM_ADDRESS_LEVEL_CONFIG`.
+Other than that, the ranks can be freely assigned via the JSON file
+defined with `CONST_Address_Level_Config` according to their type and
+the country they are in.
 
 The address level configuration must consist of an array of configuration
 entries, each containing a tag definition and an optional country array:

docs/develop/Testing.md

@@ -21,15 +21,14 @@ This test directory is sturctured as follows:
   | +-  api        Tests for API endpoints (search, reverse, etc.)
   |
   +-   php         PHP unit tests
-  +-   python      Python unit tests
   +-   scenes      Geometry test data
   +-   testdb      Base data for generating API test database
 ```
 
 ## PHP Unit Tests (`test/php`)
 
-Unit tests for PHP code can be found in the `php/` directory. They test selected
-PHP functions. Very low coverage.
+Unit tests can be found in the php/ directory. They test selected php functions.
+Very low coverage.
 
 To execute the test suite run
 
@@ -37,26 +36,11 @@ To execute the test suite run
     UNIT_TEST_DSN='pgsql:dbname=nominatim_unit_tests' phpunit ../
 
 It will read phpunit.xml which points to the library, test path, bootstrap
-strip and sets other parameters.
+strip and set other parameters.
 
 It will use (and destroy) a local database 'nominatim_unit_tests'. You can set
 a different connection string with e.g. UNIT_TEST_DSN='pgsql:dbname=foo_unit_tests'.
 
-## Python Unit Tests (`test/python`)
-
-Unit tests for Python code can be found in the `python/` directory. The goal is
-to have complete coverage of the Python library in `nominatim`.
-
-To execute the tests run
-
-    py.test-3 test/python
-
-or
-
-    pytest test/python
-
-The name of the pytest binary depends on your installation.
-
 ## BDD Functional Tests (`test/bdd`)
 
 Functional tests are written as BDD instructions. For more information on
@@ -83,17 +67,17 @@ The tests can be configured with a set of environment variables (`behave -D key=
                    the test databases (db tests)
  * `TEST_DB` - name of test database (db tests)
  * `API_TEST_DB` - name of the database containing the API test data (api tests)
- * `API_TEST_FILE` - OSM file to be imported into the API test database (api tests)
  * `DB_HOST` - (optional) hostname of database host
  * `DB_PORT` - (optional) port of database on host
  * `DB_USER` - (optional) username of database login
  * `DB_PASS` - (optional) password for database login
  * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim
                           module shared library file
- * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused
-                       during the next run. Reusing the base templates speeds
-                       up tests considerably but might lead to outdated errors
-                       for some changes in the database layout.
+ * `TEST_SETTINGS_TEMPLATE` - file to write temporary Nominatim settings to
+ * `REMOVE_TEMPLATE` - if true, the template database will not be reused during
+                       the next run. Reusing the base templates speeds up tests
+                       considerably but might lead to outdated errors for some
+                       changes in the database layout.
  * `KEEP_TEST_DB` - if true, the test database will not be dropped after a test
                     is finished. Should only be used if one single scenario is
                     run, otherwise the result is undefined.
@@ -105,20 +89,23 @@ feature of behave which comes in handy when writing new tests.
 ### API Tests (`test/bdd/api`)
 
 These tests are meant to test the different API endpoints and their parameters.
-They require to import several datasets into a test database. This is normally
-done automatically during setup of the test. The API test database is then
-kept around and reused in subsequent runs of behave. Use `behave -DREMOVE_TEMPLATE`
-to force a reimport of the database.
+They require to import several datasets into a test database.
+See the [Development Setup chapter](Development-Environment.md#preparing-the-test-database)
+for instructions on how to set up this database.
 
-The official test dataset is saved in the file `test/testdb/apidb-test-data.pbf`
-and compromises the following data:
+The official test dataset was derived from the 180924 planet (note: such
+file no longer exists at https://planet.openstreetmap.org/planet/2018/).
+Newer planets are likely to work as well but you may see isolated test
+failures where the data has changed.
 
- * Geofabrik extract of Liechtenstein
- * extract of Autauga country, Alabama, US (for tests against Tiger data)
- * additional data from `test/testdb/additional_api_test.data.osm`
+The official test dataset can always be downloaded from
+[nominatim.org](https://www.nominatim.org/data/test/nominatim-api-testdata.pbf)
+To recreate the input data for the test database run:
 
-API tests should only be testing the functionality of the website PHP code.
-Most tests should be formulated as BDD DB creation tests (see below) instead.
+```
+wget https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-180924.osm.pbf
+osmconvert planet-180924.osm.pbf -B=test/testdb/testdb.polys -o=testdb.pbf
+```
 
 #### Code Coverage
 
@@ -153,7 +140,3 @@ needs superuser rights for postgres.
 
 These tests check that data is imported correctly into the place table. They
 use the same template database as the DB Creation tests, so the same remarks apply.
-
-Note that most testing of the gazetteer output of osm2pgsql is done in the tests
-of osm2pgsql itself. The BDD tests are just there to ensure compatibility of
-the osm2pgsql and Nominatim code.

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim Documentation
+site_name: Nominatim 3.6.0
 theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org
@@ -19,8 +19,6 @@ pages:
         - 'Import' : 'admin/Import.md'
         - 'Update' : 'admin/Update.md'
         - 'Deploy' : 'admin/Deployment.md'
-        - 'Customize Imports' : 'admin/Customization.md'
-        - 'Tokenizers' : 'admin/Tokenizers.md'
         - 'Nominatim UI'  : 'admin/Setup-Nominatim-UI.md'
         - 'Advanced Installations' : 'admin/Advanced-Installations.md'
         - 'Migration from older Versions' : 'admin/Migration.md'

lib-php/SearchPosition.php

@@ -1,87 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-/**
- * Description of the position of a token within a query.
- */
-class SearchPosition
-{
-    private $sPhraseType;
-
-    private $iPhrase;
-    private $iNumPhrases;
-
-    private $iToken;
-    private $iNumTokens;
-
-
-    public function __construct($sPhraseType, $iPhrase, $iNumPhrases)
-    {
-        $this->sPhraseType = $sPhraseType;
-        $this->iPhrase = $iPhrase;
-        $this->iNumPhrases = $iNumPhrases;
-    }
-
-    public function setTokenPosition($iToken, $iNumTokens)
-    {
-        $this->iToken = $iToken;
-        $this->iNumTokens = $iNumTokens;
-    }
-
-    /**
-     * Check if the phrase can be of the given type.
-     *
-     * @param string  $sType  Type of phrse requested.
-     *
-     * @return True if the phrase is untyped or of the given type.
-     */
-    public function maybePhrase($sType)
-    {
-        return $this->sPhraseType == '' || $this->sPhraseType == $sType;
-    }
-
-    /**
-     * Check if the phrase is exactly of the given type.
-     *
-     * @param string  $sType  Type of phrse requested.
-     *
-     * @return True if the phrase of the given type.
-     */
-    public function isPhrase($sType)
-    {
-        return $this->sPhraseType == $sType;
-    }
-
-    /**
-     * Return true if the token is the very first in the query.
-     */
-    public function isFirstToken()
-    {
-        return $this->iPhrase == 0 && $this->iToken == 0;
-    }
-
-    /**
-     * Check if the token is the final one in the query.
-     */
-    public function isLastToken()
-    {
-        return $this->iToken + 1 == $this->iNumTokens && $this->iPhrase + 1 == $this->iNumPhrases;
-    }
-
-    /**
-     * Check if the current token is part of the first phrase in the query.
-     */
-    public function isFirstPhrase()
-    {
-        return $this->iPhrase == 0;
-    }
-
-    /**
-     * Get the phrase position in the query.
-     */
-    public function getPhrase()
-    {
-        return $this->iPhrase;
-    }
-}

lib-php/Status.php

@@ -1,51 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-require_once(CONST_TokenizerDir.'/tokenizer.php');
-
-use Exception;
-
-class Status
-{
-    protected $oDB;
-
-    public function __construct(&$oDB)
-    {
-        $this->oDB =& $oDB;
-    }
-
-    public function status()
-    {
-        if (!$this->oDB) {
-            throw new Exception('No database', 700);
-        }
-
-        try {
-            $this->oDB->connect();
-        } catch (\Nominatim\DatabaseError $e) {
-            throw new Exception('Database connection failed', 700);
-        }
-
-        $oTokenizer = new \Nominatim\Tokenizer($this->oDB);
-        $oTokenizer->checkStatus();
-    }
-
-    public function dataDate()
-    {
-        $sSQL = 'SELECT EXTRACT(EPOCH FROM lastimportdate) FROM import_status LIMIT 1';
-        $iDataDateEpoch = $this->oDB->getOne($sSQL);
-
-        if ($iDataDateEpoch === false) {
-            throw new Exception('Import date is not available', 705);
-        }
-
-        return $iDataDateEpoch;
-    }
-
-    public function databaseVersion()
-    {
-        $sSQL = 'SELECT value FROM nominatim_properties WHERE property = \'database_version\'';
-        return $this->oDB->getOne($sSQL);
-    }
-}

lib-php/TokenCountry.php

@@ -1,72 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-/**
- * A country token.
- */
-class Country
-{
-    /// Database word id, if available.
-    private $iId;
-    /// Two-letter country code (lower-cased).
-    private $sCountryCode;
-
-    public function __construct($iId, $sCountryCode)
-    {
-        $this->iId = $iId;
-        $this->sCountryCode = $sCountryCode;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasCountry() && $oPosition->maybePhrase('country');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $oNewSearch = $oSearch->clone($oPosition->isLastToken() ? 1 : 6);
-        $oNewSearch->setCountry($this->sCountryCode);
-
-        return array($oNewSearch);
-    }
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'country',
-                'Info' => $this->sCountryCode
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'C';
-    }
-}

lib-php/TokenHousenumber.php

@@ -1,108 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-/**
- * A house number token.
- */
-class HouseNumber
-{
-    /// Database word id, if available.
-    private $iId;
-    /// Normalized house number.
-    private $sToken;
-
-    public function __construct($iId, $sToken)
-    {
-        $this->iId = $iId;
-        $this->sToken = $sToken;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasHousenumber()
-               && !$oSearch->hasOperator(\Nominatim\Operator::POSTCODE)
-               && $oPosition->maybePhrase('street');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $aNewSearches = array();
-
-        // sanity check: if the housenumber is not mainly made
-        // up of numbers, add a penalty
-        $iSearchCost = 1;
-        if (preg_match('/\\d/', $this->sToken) === 0
-            || preg_match_all('/[^0-9]/', $this->sToken, $aMatches) > 2) {
-            $iSearchCost++;
-        }
-        if (!$oSearch->hasOperator(\Nominatim\Operator::NONE)) {
-            $iSearchCost++;
-        }
-        if (empty($this->iId)) {
-            $iSearchCost++;
-        }
-        // also must not appear in the middle of the address
-        if ($oSearch->hasAddress() || $oSearch->hasPostcode()) {
-            $iSearchCost++;
-        }
-
-        $oNewSearch = $oSearch->clone($iSearchCost);
-        $oNewSearch->setHousenumber($this->sToken);
-        $aNewSearches[] = $oNewSearch;
-
-        // Housenumbers may appear in the name when the place has its own
-        // address terms.
-        if ($this->iId !== null
-            && ($oSearch->getNamePhrase() >= 0 || !$oSearch->hasName())
-            && !$oSearch->hasAddress()
-        ) {
-            $oNewSearch = $oSearch->clone($iSearchCost);
-            $oNewSearch->setHousenumberAsName($this->iId);
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        return $aNewSearches;
-    }
-
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'house number',
-                'Info' => array('nr' => $this->sToken)
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'H';
-    }
-}

lib-php/TokenList.php

@@ -1,126 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-require_once(CONST_LibDir.'/TokenCountry.php');
-require_once(CONST_LibDir.'/TokenHousenumber.php');
-require_once(CONST_LibDir.'/TokenPostcode.php');
-require_once(CONST_LibDir.'/TokenSpecialTerm.php');
-require_once(CONST_LibDir.'/TokenWord.php');
-require_once(CONST_LibDir.'/TokenPartial.php');
-require_once(CONST_LibDir.'/SpecialSearchOperator.php');
-
-/**
- * Saves information about the tokens that appear in a search query.
- *
- * Tokens are sorted by their normalized form, the token word. There are different
- * kinds of tokens, represented by different Token* classes. Note that
- * tokens do not have a common base class. All tokens need to have a field
- * with the word id that points to an entry in the `word` database table
- * but otherwise the information saved about a token can be very different.
- */
-class TokenList
-{
-    // List of list of tokens indexed by their word_token.
-    private $aTokens = array();
-
-
-    /**
-     * Return total number of tokens.
-     *
-     * @return Integer
-     */
-    public function count()
-    {
-        return count($this->aTokens);
-    }
-
-    /**
-     * Check if there are tokens for the given token word.
-     *
-     * @param string $sWord Token word to look for.
-     *
-     * @return bool True if there is one or more token for the token word.
-     */
-    public function contains($sWord)
-    {
-        return isset($this->aTokens[$sWord]);
-    }
-
-    /**
-     * Check if there are partial or full tokens for the given word.
-     *
-     * @param string $sWord Token word to look for.
-     *
-     * @return bool True if there is one or more token for the token word.
-     */
-    public function containsAny($sWord)
-    {
-        return isset($this->aTokens[$sWord]);
-    }
-
-    /**
-     * Get the list of tokens for the given token word.
-     *
-     * @param string $sWord Token word to look for.
-     *
-     * @return object[] Array of tokens for the given token word or an
-     *                  empty array if no tokens could be found.
-     */
-    public function get($sWord)
-    {
-        return isset($this->aTokens[$sWord]) ? $this->aTokens[$sWord] : array();
-    }
-
-    public function getFullWordIDs()
-    {
-        $ids = array();
-
-        foreach ($this->aTokens as $aTokenList) {
-            foreach ($aTokenList as $oToken) {
-                if (is_a($oToken, '\Nominatim\Token\Word')) {
-                    $ids[$oToken->getId()] = $oToken->getId();
-                }
-            }
-        }
-
-        return $ids;
-    }
-
-    /**
-     * Add a new token for the given word.
-     *
-     * @param string $sWord  Word the token describes.
-     * @param object $oToken Token object to add.
-     *
-     * @return void
-     */
-    public function addToken($sWord, $oToken)
-    {
-        if (isset($this->aTokens[$sWord])) {
-            $this->aTokens[$sWord][] = $oToken;
-        } else {
-            $this->aTokens[$sWord] = array($oToken);
-        }
-    }
-
-    public function debugTokenByWordIdList()
-    {
-        $aWordsIDs = array();
-        foreach ($this->aTokens as $sToken => $aWords) {
-            foreach ($aWords as $aToken) {
-                $iId = $aToken->getId();
-                if ($iId !== null) {
-                    $aWordsIDs[$iId] = '#'.$sToken.'('.$aToken->debugCode().' '.$iId.')#';
-                }
-            }
-        }
-
-        return $aWordsIDs;
-    }
-
-    public function debugInfo()
-    {
-        return $this->aTokens;
-    }
-}

lib-php/TokenPartial.php

@@ -1,118 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-/**
- * A standard word token.
- */
-class Partial
-{
-    /// Database word id, if applicable.
-    private $iId;
-    /// Number of appearances in the database.
-    private $iSearchNameCount;
-    /// True, if the token consists exclusively of digits and spaces.
-    private $bNumberToken;
-
-    public function __construct($iId, $sToken, $iSearchNameCount)
-    {
-        $this->iId = $iId;
-        $this->bNumberToken = (bool) preg_match('#^[0-9 ]+$#', $sToken);
-        $this->iSearchNameCount = $iSearchNameCount;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oPosition->isPhrase('country');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $aNewSearches = array();
-
-        // Partial token in Address.
-        if (($oPosition->isPhrase('') || !$oPosition->isFirstPhrase())
-            && $oSearch->hasName()
-        ) {
-            $iSearchCost = $this->bNumberToken ? 2 : 1;
-            if ($this->iSearchNameCount >= CONST_Max_Word_Frequency) {
-                $iSearchCost += 1;
-            }
-
-            $oNewSearch = $oSearch->clone($iSearchCost);
-            $oNewSearch->addAddressToken(
-                $this->iId,
-                $this->iSearchNameCount < CONST_Max_Word_Frequency
-            );
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        // Partial token in Name.
-        if ((!$oSearch->hasPostcode() && !$oSearch->hasAddress())
-            && (!$oSearch->hasName(true)
-                || $oSearch->getNamePhrase() == $oPosition->getPhrase())
-        ) {
-            $iSearchCost = 1;
-            if (!$oSearch->hasName(true)) {
-                $iSearchCost += 1;
-            }
-            if ($this->bNumberToken) {
-                $iSearchCost += 1;
-            }
-
-            $oNewSearch = $oSearch->clone($iSearchCost);
-            $oNewSearch->addPartialNameToken(
-                $this->iId,
-                $this->iSearchNameCount < CONST_Max_Word_Frequency,
-                $oPosition->getPhrase()
-            );
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        return $aNewSearches;
-    }
-
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'partial',
-                'Info' => array(
-                           'count' => $this->iSearchNameCount
-                          )
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'w';
-    }
-}

lib-php/TokenPostcode.php

@@ -1,98 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-/**
- * A postcode token.
- */
-class Postcode
-{
-    /// Database word id, if available.
-    private $iId;
-    /// Full nomralized postcode (upper cased).
-    private $sPostcode;
-    // Optional country code the postcode belongs to (currently unused).
-    private $sCountryCode;
-
-    public function __construct($iId, $sPostcode, $sCountryCode = '')
-    {
-        $this->iId = $iId;
-        $this->sPostcode = $sPostcode;
-        $this->sCountryCode = empty($sCountryCode) ? '' : $sCountryCode;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasPostcode() && $oPosition->maybePhrase('postalcode');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $aNewSearches = array();
-
-        // If we have structured search or this is the first term,
-        // make the postcode the primary search element.
-        if ($oSearch->hasOperator(\Nominatim\Operator::NONE) && $oPosition->isFirstToken()) {
-            $oNewSearch = $oSearch->clone(1);
-            $oNewSearch->setPostcodeAsName($this->iId, $this->sPostcode);
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        // If we have a structured search or this is not the first term,
-        // add the postcode as an addendum.
-        if (!$oSearch->hasOperator(\Nominatim\Operator::POSTCODE)
-            && ($oPosition->isPhrase('postalcode') || $oSearch->hasName())
-        ) {
-            $iPenalty = 1;
-            if (strlen($this->sPostcode) < 4) {
-                $iPenalty += 4 - strlen($this->sPostcode);
-            }
-            $oNewSearch = $oSearch->clone($iPenalty);
-            $oNewSearch->setPostcode($this->sPostcode);
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        return $aNewSearches;
-    }
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'postcode',
-                'Info' => $this->sPostcode.'('.$this->sCountryCode.')'
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'P';
-    }
-}

lib-php/TokenSpecialTerm.php

@@ -1,102 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-require_once(CONST_LibDir.'/SpecialSearchOperator.php');
-
-/**
- * A word token describing a place type.
- */
-class SpecialTerm
-{
-    /// Database word id, if applicable.
-    private $iId;
-    /// Class (or OSM tag key) of the place to look for.
-    private $sClass;
-    /// Type (or OSM tag value) of the place to look for.
-    private $sType;
-    /// Relationship of the operator to the object (see Operator class).
-    private $iOperator;
-
-    public function __construct($iID, $sClass, $sType, $iOperator)
-    {
-        $this->iId = $iID;
-        $this->sClass = $sClass;
-        $this->sType = $sType;
-        $this->iOperator = $iOperator;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasOperator() && $oPosition->isPhrase('');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $iSearchCost = 2;
-
-        $iOp = $this->iOperator;
-        if ($iOp == \Nominatim\Operator::NONE) {
-            if ($oSearch->hasName() || $oSearch->getContext()->isBoundedSearch()) {
-                $iOp = \Nominatim\Operator::NAME;
-            } else {
-                $iOp = \Nominatim\Operator::NEAR;
-            }
-            $iSearchCost += 2;
-        } elseif (!$oPosition->isFirstToken() && !$oPosition->isLastToken()) {
-            $iSearchCost += 2;
-        }
-        if ($oSearch->hasHousenumber()) {
-            $iSearchCost ++;
-        }
-
-        $oNewSearch = $oSearch->clone($iSearchCost);
-        $oNewSearch->setPoiSearch($iOp, $this->sClass, $this->sType);
-
-        return array($oNewSearch);
-    }
-
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'special term',
-                'Info' => array(
-                           'class' => $this->sClass,
-                           'type' => $this->sType,
-                           'operator' => \Nominatim\Operator::toString($this->iOperator)
-                          )
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'S';
-    }
-}

lib-php/TokenWord.php

@@ -1,102 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-/**
- * A standard word token.
- */
-class Word
-{
-    /// Database word id, if applicable.
-    private $iId;
-    /// Number of appearances in the database.
-    private $iSearchNameCount;
-    /// Number of terms in the word.
-    private $iTermCount;
-
-    public function __construct($iId, $iSearchNameCount, $iTermCount)
-    {
-        $this->iId = $iId;
-        $this->iSearchNameCount = $iSearchNameCount;
-        $this->iTermCount = $iTermCount;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oPosition->isPhrase('country');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        // Full words can only be a name if they appear at the beginning
-        // of the phrase. In structured search the name must forcably in
-        // the first phrase. In unstructured search it may be in a later
-        // phrase when the first phrase is a house number.
-        if ($oSearch->hasName()
-            || !($oPosition->isFirstPhrase() || $oPosition->isPhrase(''))
-        ) {
-            if ($this->iTermCount > 1
-                && ($oPosition->isPhrase('') || !$oPosition->isFirstPhrase())
-            ) {
-                $oNewSearch = $oSearch->clone(1);
-                $oNewSearch->addAddressToken($this->iId);
-
-                return array($oNewSearch);
-            }
-        } elseif (!$oSearch->hasName(true)) {
-            $oNewSearch = $oSearch->clone(1);
-            $oNewSearch->addNameToken(
-                $this->iId,
-                CONST_Search_NameOnlySearchFrequencyThreshold
-                && $this->iSearchNameCount
-                          < CONST_Search_NameOnlySearchFrequencyThreshold
-            );
-
-            return array($oNewSearch);
-        }
-
-        return array();
-    }
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'word',
-                'Info' => array(
-                           'count' => $this->iSearchNameCount,
-                           'terms' => $this->iTermCount
-                          )
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'W';
-    }
-}

lib-php/dotenv_loader.php

@@ -1,13 +0,0 @@
-<?php
-
-require('Symfony/Component/Dotenv/autoload.php');
-
-function loadDotEnv()
-{
-    $dotenv = new \Symfony\Component\Dotenv\Dotenv();
-    $dotenv->load(CONST_ConfigDir.'/env.defaults');
-
-    if (file_exists('.env')) {
-        $dotenv->load('.env');
-    }
-}

lib-php/init-cmd.php

@@ -1,5 +0,0 @@
-<?php
-
-require_once('init.php');
-require_once('cmd.php');
-require_once('DebugNone.php');

lib-php/init.php

@@ -1,4 +0,0 @@
-<?php
-
-require_once(CONST_LibDir.'/lib.php');
-require_once(CONST_LibDir.'/DB.php');

lib-php/migration/PhraseSettingsToJson.php

@@ -1,21 +0,0 @@
-<?php
-
-$phpPhraseSettingsFile = $argv[1];
-$jsonPhraseSettingsFile = dirname($phpPhraseSettingsFile).'/'.basename($phpPhraseSettingsFile, '.php').'.json';
-
-if (file_exists($phpPhraseSettingsFile) && !file_exists($jsonPhraseSettingsFile)) {
-    include $phpPhraseSettingsFile;
-
-    $data = array();
-
-    if (isset($aTagsBlacklist)) {
-        $data['blackList'] = $aTagsBlacklist;
-    }
-    if (isset($aTagsWhitelist)) {
-        $data['whiteList'] = $aTagsWhitelist;
-    }
-
-    $jsonFile = fopen($jsonPhraseSettingsFile, 'w');
-    fwrite($jsonFile, json_encode($data));
-    fclose($jsonFile);
-}

lib-php/output.php

@@ -1,30 +0,0 @@
-<?php
-
-
-function formatOSMType($sType, $bIncludeExternal = true)
-{
-    if ($sType == 'N') {
-        return 'node';
-    }
-    if ($sType == 'W') {
-        return 'way';
-    }
-    if ($sType == 'R') {
-        return 'relation';
-    }
-
-    if (!$bIncludeExternal) {
-        return '';
-    }
-
-    if ($sType == 'T') {
-        return 'way';
-    }
-    if ($sType == 'I') {
-        return 'way';
-    }
-
-    // not handled: P, L
-
-    return '';
-}

lib-php/setup_functions.php

@@ -1,19 +0,0 @@
-<?php
-
-function getOsm2pgsqlBinary()
-{
-    $sBinary = getSetting('OSM2PGSQL_BINARY');
-
-    return $sBinary ? $sBinary : CONST_Default_Osm2pgsql;
-}
-
-function getImportStyle()
-{
-    $sStyle = getSetting('IMPORT_STYLE');
-
-    if (in_array($sStyle, array('admin', 'street', 'address', 'full', 'extratags'))) {
-        return CONST_ConfigDir.'/import-'.$sStyle.'.style';
-    }
-
-    return $sStyle;
-}

lib-php/tokenizer/legacy_icu_tokenizer.php

@@ -1,246 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-class Tokenizer
-{
-    private $oDB;
-
-    private $oNormalizer;
-    private $oTransliterator;
-    private $aCountryRestriction;
-
-    public function __construct(&$oDB)
-    {
-        $this->oDB =& $oDB;
-        $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
-        $this->oTransliterator = \Transliterator::createFromRules(CONST_Transliteration);
-    }
-
-    public function checkStatus()
-    {
-        $sSQL = 'SELECT word_id FROM word limit 1';
-        $iWordID = $this->oDB->getOne($sSQL);
-        if ($iWordID === false) {
-            throw new \Exception('Query failed', 703);
-        }
-        if (!$iWordID) {
-            throw new \Exception('No value', 704);
-        }
-    }
-
-
-    public function setCountryRestriction($aCountries)
-    {
-        $this->aCountryRestriction = $aCountries;
-    }
-
-
-    public function normalizeString($sTerm)
-    {
-        if ($this->oNormalizer === null) {
-            return $sTerm;
-        }
-
-        return $this->oNormalizer->transliterate($sTerm);
-    }
-
-    private function makeStandardWord($sTerm)
-    {
-        return trim($this->oTransliterator->transliterate(' '.$sTerm.' '));
-    }
-
-
-    public function tokensForSpecialTerm($sTerm)
-    {
-        $aResults = array();
-
-        $sSQL = "SELECT word_id, info->>'class' as class, info->>'type' as type ";
-        $sSQL .= '   FROM word WHERE word_token = :term and type = \'S\'';
-
-        Debug::printVar('Term', $sTerm);
-        Debug::printSQL($sSQL);
-        $aSearchWords = $this->oDB->getAll($sSQL, array(':term' => $this->makeStandardWord($sTerm)));
-
-        Debug::printVar('Results', $aSearchWords);
-
-        foreach ($aSearchWords as $aSearchTerm) {
-            $aResults[] = new \Nominatim\Token\SpecialTerm(
-                $aSearchTerm['word_id'],
-                $aSearchTerm['class'],
-                $aSearchTerm['type'],
-                \Nominatim\Operator::TYPE
-            );
-        }
-
-        Debug::printVar('Special term tokens', $aResults);
-
-        return $aResults;
-    }
-
-
-    public function extractTokensFromPhrases(&$aPhrases)
-    {
-        $sNormQuery = '';
-        $aWordLists = array();
-        $aTokens = array();
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase());
-            $sPhrase = $this->makeStandardWord($oPhrase->getPhrase());
-            Debug::printVar('Phrase', $sPhrase);
-            if (strlen($sPhrase) > 0) {
-                $aWords = explode(' ', $sPhrase);
-                Tokenizer::addTokens($aTokens, $aWords);
-                $aWordLists[] = $aWords;
-            } else {
-                $aWordLists[] = array();
-            }
-        }
-
-        Debug::printVar('Tokens', $aTokens);
-        Debug::printVar('WordLists', $aWordLists);
-
-        $oValidTokens = $this->computeValidTokens($aTokens, $sNormQuery);
-
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $oPhrase->computeWordSets($aWordLists[$iPhrase], $oValidTokens);
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function computeValidTokens($aTokens, $sNormQuery)
-    {
-        $oValidTokens = new TokenList();
-
-        if (!empty($aTokens)) {
-            $this->addTokensFromDB($oValidTokens, $aTokens, $sNormQuery);
-
-            // Try more interpretations for Tokens that could not be matched.
-            foreach ($aTokens as $sToken) {
-                if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) {
-                    if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
-                        // US ZIP+4 codes - merge in the 5-digit ZIP code
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\Postcode(null, $aData[1], 'us')
-                        );
-                    } elseif (preg_match('/^[0-9]+$/', $sToken)) {
-                        // Unknown single word token with a number.
-                        // Assume it is a house number.
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\HouseNumber(null, trim($sToken))
-                        );
-                    }
-                }
-            }
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery)
-    {
-        // Check which tokens we have, get the ID numbers
-        $sSQL = 'SELECT word_id, word_token, type, word,';
-        $sSQL .= "      info->>'op' as operator,";
-        $sSQL .= "      info->>'class' as class, info->>'type' as ctype,";
-        $sSQL .= "      info->>'count' as count";
-        $sSQL .= ' FROM word WHERE word_token in (';
-        $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';
-
-        Debug::printSQL($sSQL);
-
-        $aDBWords = $this->oDB->getAll($sSQL, null, 'Could not get word tokens.');
-
-        foreach ($aDBWords as $aWord) {
-            $iId = (int) $aWord['word_id'];
-            $sTok = $aWord['word_token'];
-
-            switch ($aWord['type']) {
-                case 'C':  // country name tokens
-                    if ($aWord['word'] !== null
-                        && (!$this->aCountryRestriction
-                            || in_array($aWord['word'], $this->aCountryRestriction))
-                    ) {
-                        $oValidTokens->addToken(
-                            $sTok,
-                            new Token\Country($iId, $aWord['word'])
-                        );
-                    }
-                    break;
-                case 'H':  // house number tokens
-                    $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $aWord['word_token']));
-                    break;
-                case 'P':  // postcode tokens
-                    // Postcodes are not normalized, so they may have content
-                    // that makes SQL injection possible. Reject postcodes
-                    // that would need special escaping.
-                    if ($aWord['word'] !== null
-                        && pg_escape_string($aWord['word']) == $aWord['word']
-                    ) {
-                        $sNormPostcode = $this->normalizeString($aWord['word']);
-                        if (strpos($sNormQuery, $sNormPostcode) !== false) {
-                            $oValidTokens->addToken(
-                                $sTok,
-                                new Token\Postcode($iId, $aWord['word'], null)
-                            );
-                        }
-                    }
-                    break;
-                case 'S':  // tokens for classification terms (special phrases)
-                    if ($aWord['class'] !== null && $aWord['ctype'] !== null) {
-                        $oValidTokens->addToken($sTok, new Token\SpecialTerm(
-                            $iId,
-                            $aWord['class'],
-                            $aWord['ctype'],
-                            (isset($aWord['operator'])) ? Operator::NEAR : Operator::NONE
-                        ));
-                    }
-                    break;
-                case 'W': // full-word tokens
-                    $oValidTokens->addToken($sTok, new Token\Word(
-                        $iId,
-                        (int) $aWord['count'],
-                        substr_count($aWord['word_token'], ' ')
-                    ));
-                    break;
-                case 'w':  // partial word terms
-                    $oValidTokens->addToken($sTok, new Token\Partial(
-                        $iId,
-                        $aWord['word_token'],
-                        (int) $aWord['count']
-                    ));
-                    break;
-                default:
-                    break;
-            }
-        }
-    }
-
-
-    /**
-     * Add the tokens from this phrase to the given list of tokens.
-     *
-     * @param string[] $aTokens List of tokens to append.
-     *
-     * @return void
-     */
-    private static function addTokens(&$aTokens, $aWords)
-    {
-        $iNumWords = count($aWords);
-
-        for ($i = 0; $i < $iNumWords; $i++) {
-            $sPhrase = $aWords[$i];
-            $aTokens[$sPhrase] = $sPhrase;
-
-            for ($j = $i + 1; $j < $iNumWords; $j++) {
-                $sPhrase .= ' '.$aWords[$j];
-                $aTokens[$sPhrase] = $sPhrase;
-            }
-        }
-    }
-}

lib-php/tokenizer/legacy_tokenizer.php

@@ -1,266 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-class Tokenizer
-{
-    private $oDB;
-
-    private $oNormalizer = null;
-    private $aCountryRestriction = null;
-
-    public function __construct(&$oDB)
-    {
-        $this->oDB =& $oDB;
-        $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
-    }
-
-    public function checkStatus()
-    {
-        $sStandardWord = $this->oDB->getOne("SELECT make_standard_name('a')");
-        if ($sStandardWord === false) {
-            throw new \Exception('Module failed', 701);
-        }
-
-        if ($sStandardWord != 'a') {
-            throw new \Exception('Module call failed', 702);
-        }
-
-        $sSQL = "SELECT word_id FROM word WHERE word_token IN (' a')";
-        $iWordID = $this->oDB->getOne($sSQL);
-        if ($iWordID === false) {
-            throw new \Exception('Query failed', 703);
-        }
-        if (!$iWordID) {
-            throw new \Exception('No value', 704);
-        }
-    }
-
-
-    public function setCountryRestriction($aCountries)
-    {
-        $this->aCountryRestriction = $aCountries;
-    }
-
-
-    public function normalizeString($sTerm)
-    {
-        if ($this->oNormalizer === null) {
-            return $sTerm;
-        }
-
-        return $this->oNormalizer->transliterate($sTerm);
-    }
-
-
-    public function tokensForSpecialTerm($sTerm)
-    {
-        $aResults = array();
-
-        $sSQL = 'SELECT word_id, class, type FROM word ';
-        $sSQL .= '   WHERE word_token = \' \' || make_standard_name(:term)';
-        $sSQL .= '   AND class is not null AND class not in (\'place\')';
-
-        Debug::printVar('Term', $sTerm);
-        Debug::printSQL($sSQL);
-        $aSearchWords = $this->oDB->getAll($sSQL, array(':term' => $sTerm));
-
-        Debug::printVar('Results', $aSearchWords);
-
-        foreach ($aSearchWords as $aSearchTerm) {
-            $aResults[] = new \Nominatim\Token\SpecialTerm(
-                $aSearchTerm['word_id'],
-                $aSearchTerm['class'],
-                $aSearchTerm['type'],
-                \Nominatim\Operator::TYPE
-            );
-        }
-
-        Debug::printVar('Special term tokens', $aResults);
-
-        return $aResults;
-    }
-
-
-    public function extractTokensFromPhrases(&$aPhrases)
-    {
-        // First get the normalized version of all phrases
-        $sNormQuery = '';
-        $sSQL = 'SELECT ';
-        $aParams = array();
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase());
-            $sSQL .= 'make_standard_name(:' .$iPhrase.') as p'.$iPhrase.',';
-            $aParams[':'.$iPhrase] = $oPhrase->getPhrase();
-        }
-        $sSQL = substr($sSQL, 0, -1);
-
-        Debug::printSQL($sSQL);
-        Debug::printVar('SQL parameters', $aParams);
-
-        $aNormPhrases = $this->oDB->getRow($sSQL, $aParams);
-
-        Debug::printVar('SQL result', $aNormPhrases);
-
-        // now compute all possible tokens
-        $aWordLists = array();
-        $aTokens = array();
-        foreach ($aNormPhrases as $sPhrase) {
-            if (strlen($sPhrase) > 0) {
-                $aWords = explode(' ', $sPhrase);
-                Tokenizer::addTokens($aTokens, $aWords);
-                $aWordLists[] = $aWords;
-            } else {
-                $aWordLists[] = array();
-            }
-        }
-
-        Debug::printVar('Tokens', $aTokens);
-        Debug::printVar('WordLists', $aWordLists);
-
-        $oValidTokens = $this->computeValidTokens($aTokens, $sNormQuery);
-
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $oPhrase->computeWordSets($aWordLists[$iPhrase], $oValidTokens);
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function computeValidTokens($aTokens, $sNormQuery)
-    {
-        $oValidTokens = new TokenList();
-
-        if (!empty($aTokens)) {
-            $this->addTokensFromDB($oValidTokens, $aTokens, $sNormQuery);
-
-            // Try more interpretations for Tokens that could not be matched.
-            foreach ($aTokens as $sToken) {
-                if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) {
-                    if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
-                        // US ZIP+4 codes - merge in the 5-digit ZIP code
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\Postcode(null, $aData[1], 'us')
-                        );
-                    } elseif (preg_match('/^[0-9]+$/', $sToken)) {
-                        // Unknown single word token with a number.
-                        // Assume it is a house number.
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\HouseNumber(null, trim($sToken))
-                        );
-                    }
-                }
-            }
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery)
-    {
-        // Check which tokens we have, get the ID numbers
-        $sSQL = 'SELECT word_id, word_token, word, class, type, country_code,';
-        $sSQL .= ' operator, coalesce(search_name_count, 0) as count';
-        $sSQL .= ' FROM word WHERE word_token in (';
-        $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';
-
-        Debug::printSQL($sSQL);
-
-        $aDBWords = $this->oDB->getAll($sSQL, null, 'Could not get word tokens.');
-
-        foreach ($aDBWords as $aWord) {
-            $oToken = null;
-            $iId = (int) $aWord['word_id'];
-
-            if ($aWord['class']) {
-                // Special terms need to appear in their normalized form.
-                // (postcodes are not normalized in the word table)
-                $sNormWord = $this->normalizeString($aWord['word']);
-                if ($aWord['word'] && strpos($sNormQuery, $sNormWord) === false) {
-                    continue;
-                }
-
-                if ($aWord['class'] == 'place' && $aWord['type'] == 'house') {
-                    $oToken = new Token\HouseNumber($iId, trim($aWord['word_token']));
-                } elseif ($aWord['class'] == 'place' && $aWord['type'] == 'postcode') {
-                    if ($aWord['word']
-                        && pg_escape_string($aWord['word']) == $aWord['word']
-                    ) {
-                        $oToken = new Token\Postcode(
-                            $iId,
-                            $aWord['word'],
-                            $aWord['country_code']
-                        );
-                    }
-                } else {
-                    // near and in operator the same at the moment
-                    $oToken = new Token\SpecialTerm(
-                        $iId,
-                        $aWord['class'],
-                        $aWord['type'],
-                        $aWord['operator'] ? Operator::NEAR : Operator::NONE
-                    );
-                }
-            } elseif ($aWord['country_code']) {
-                // Filter country tokens that do not match restricted countries.
-                if (!$this->aCountryRestriction
-                    || in_array($aWord['country_code'], $this->aCountryRestriction)
-                ) {
-                    $oToken = new Token\Country($iId, $aWord['country_code']);
-                }
-            } elseif ($aWord['word_token'][0] == ' ') {
-                $oToken = new Token\Word(
-                    $iId,
-                    (int) $aWord['count'],
-                    substr_count($aWord['word_token'], ' ')
-                );
-            // For backward compatibility: ignore all partial tokens with more
-            // than one word.
-            } elseif (strpos($aWord['word_token'], ' ') === false) {
-                $oToken = new Token\Partial(
-                    $iId,
-                    $aWord['word_token'],
-                    (int) $aWord['count']
-                );
-            }
-
-            if ($oToken) {
-                // remove any leading spaces
-                if ($aWord['word_token'][0] == ' ') {
-                    $oValidTokens->addToken(substr($aWord['word_token'], 1), $oToken);
-                } else {
-                    $oValidTokens->addToken($aWord['word_token'], $oToken);
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Add the tokens from this phrase to the given list of tokens.
-     *
-     * @param string[] $aTokens List of tokens to append.
-     *
-     * @return void
-     */
-    private static function addTokens(&$aTokens, $aWords)
-    {
-        $iNumWords = count($aWords);
-
-        for ($i = 0; $i < $iNumWords; $i++) {
-            $sPhrase = $aWords[$i];
-            $aTokens[' '.$sPhrase] = ' '.$sPhrase;
-            $aTokens[$sPhrase] = $sPhrase;
-
-            for ($j = $i + 1; $j < $iNumWords; $j++) {
-                $sPhrase .= ' '.$aWords[$j];
-                $aTokens[' '.$sPhrase] = ' '.$sPhrase;
-                $aTokens[$sPhrase] = $sPhrase;
-            }
-        }
-    }
-}

lib-php/website/reverse-only-search.php

@@ -1,12 +0,0 @@
-<?php
-
-require_once(CONST_LibDir.'/init-website.php');
-require_once(CONST_LibDir.'/ParameterParser.php');
-
-$oParams = new Nominatim\ParameterParser();
-
-// Format for output
-$sOutputFormat = $oParams->getSet('format', array('xml', 'json', 'jsonv2', 'geojson', 'geocodejson'), 'jsonv2');
-set_exception_handler_by_format($sOutputFormat);
-
-throw new Exception('Reverse-only import does not support forward searching.', 404);

lib-php/website/status.php

@@ -1,48 +0,0 @@
-<?php
-
-require_once(CONST_LibDir.'/init-website.php');
-require_once(CONST_LibDir.'/ParameterParser.php');
-require_once(CONST_LibDir.'/Status.php');
-
-$oParams = new Nominatim\ParameterParser();
-$sOutputFormat = $oParams->getSet('format', array('text', 'json'), 'text');
-
-$oDB = new Nominatim\DB(CONST_Database_DSN);
-
-if ($sOutputFormat == 'json') {
-    header('content-type: application/json; charset=UTF-8');
-}
-
-
-try {
-    $oStatus = new Nominatim\Status($oDB);
-    $oStatus->status();
-
-    if ($sOutputFormat == 'json') {
-        $epoch = $oStatus->dataDate();
-        $aResponse = array(
-                      'status' => 0,
-                      'message' => 'OK',
-                      'data_updated' => (new DateTime('@'.$epoch))->format(DateTime::RFC3339),
-                      'software_version' => CONST_NominatimVersion
-                     );
-        $sDatabaseVersion = $oStatus->databaseVersion();
-        if ($sDatabaseVersion) {
-            $aResponse['database_version'] = $sDatabaseVersion;
-        }
-        javascript_renderData($aResponse);
-    } else {
-        echo 'OK';
-    }
-} catch (Exception $oErr) {
-    if ($sOutputFormat == 'json') {
-        $aResponse = array(
-                      'status' => $oErr->getCode(),
-                      'message' => $oErr->getMessage()
-                     );
-        javascript_renderData($aResponse);
-    } else {
-        header('HTTP/1.0 500 Internal Server Error');
-        echo 'ERROR: '.$oErr->getMessage();
-    }
-}

lib-sql/functions.sql

@@ -1,19 +0,0 @@
-{% include('functions/utils.sql') %}
-{% include('functions/ranking.sql') %}
-{% include('functions/importance.sql') %}
-{% include('functions/address_lookup.sql') %}
-{% include('functions/interpolation.sql') %}
-
-{% if 'place' in db.tables %}
-    {% include 'functions/place_triggers.sql' %}
-{% endif %}
-
-{% if 'placex' in db.tables %}
-    {% include 'functions/placex_triggers.sql' %}
-{% endif %}
-
-{% if 'location_postcode' in db.tables %}
-    {% include 'functions/postcode_triggers.sql' %}
-{% endif %}
-
-{% include('functions/partition-functions.sql') %}

lib-sql/indices.sql

@@ -1,62 +0,0 @@
--- Indices used only during search and update.
--- These indices are created only after the indexing process is done.
-
-CREATE INDEX IF NOT EXISTS idx_place_addressline_address_place_id
-  ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}};
-
-CREATE INDEX IF NOT EXISTS idx_placex_rank_search
-  ON placex USING BTREE (rank_search) {{db.tablespace.search_index}};
-
-CREATE INDEX IF NOT EXISTS idx_placex_rank_address
-  ON placex USING BTREE (rank_address) {{db.tablespace.search_index}};
-
-CREATE INDEX IF NOT EXISTS idx_placex_parent_place_id
-  ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}}
-  WHERE parent_place_id IS NOT NULL;
-
-CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon
-  ON placex USING gist (geometry) {{db.tablespace.search_index}}
-  WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
-    AND rank_address between 4 and 25 AND type != 'postcode'
-    AND name is not null AND indexed_status = 0 AND linked_place_id is null;
-
-CREATE INDEX IF NOT EXISTS idx_osmline_parent_place_id
-  ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}};
-
-CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id
-  ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}};
-
-CREATE INDEX IF NOT EXISTS idx_postcode_postcode
-  ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}};
-
--- Indices only needed for updating.
-
-{% if not drop %}
-  CREATE INDEX IF NOT EXISTS idx_placex_pendingsector
-    ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}}
-    WHERE indexed_status > 0;
-
-  CREATE INDEX IF NOT EXISTS idx_location_area_country_place_id
-    ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}};
-
-  CREATE UNIQUE INDEX IF NOT EXISTS idx_place_osm_unique
-    ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}};
-{% endif %}
-
--- Indices only needed for search.
-
-{% if 'search_name' in db.tables %}
-  CREATE INDEX IF NOT EXISTS idx_search_name_nameaddress_vector
-    ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
-  CREATE INDEX IF NOT EXISTS idx_search_name_name_vector
-    ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
-  CREATE INDEX IF NOT EXISTS idx_search_name_centroid
-    ON search_name USING GIST (centroid) {{db.tablespace.search_index}};
-
-  {% if postgres.has_index_non_key_column %}
-    CREATE INDEX IF NOT EXISTS idx_placex_housenumber
-      ON placex USING btree (parent_place_id) INCLUDE (housenumber) WHERE housenumber is not null;
-    CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id_with_hnr
-      ON location_property_osmline USING btree(parent_place_id) INCLUDE (startnumber, endnumber);
-  {% endif %}
-{% endif %}

lib-sql/partition-tables.src.sql

@@ -1,30 +0,0 @@
-drop table IF EXISTS search_name_blank CASCADE;
-CREATE TABLE search_name_blank (
-  place_id BIGINT,
-  address_rank smallint,
-  name_vector integer[],
-  centroid GEOMETRY(Geometry, 4326)
-  );
-
-
-{% for partition in db.partitions %}
-  CREATE TABLE location_area_large_{{ partition }} () INHERITS (location_area_large) {{db.tablespace.address_data}};
-  CREATE INDEX idx_location_area_large_{{ partition }}_place_id ON location_area_large_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-  CREATE INDEX idx_location_area_large_{{ partition }}_geometry ON location_area_large_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
-
-  CREATE TABLE search_name_{{ partition }} () INHERITS (search_name_blank) {{db.tablespace.address_data}};
-  CREATE INDEX idx_search_name_{{ partition }}_place_id ON search_name_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-  CREATE INDEX idx_search_name_{{ partition }}_centroid_street ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 26 and 27;
-  CREATE INDEX idx_search_name_{{ partition }}_centroid_place ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 2 and 25;
-
-  DROP TABLE IF EXISTS location_road_{{ partition }};
-  CREATE TABLE location_road_{{ partition }} (
-    place_id BIGINT,
-    partition SMALLINT,
-    country_code VARCHAR(2),
-    geometry GEOMETRY(Geometry, 4326)
-    ) {{db.tablespace.address_data}};
-  CREATE INDEX idx_location_road_{{ partition }}_geometry ON location_road_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
-  CREATE INDEX idx_location_road_{{ partition }}_place_id ON location_road_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-
-{% endfor %}

lib-sql/postcode_tables.sql

@@ -1,15 +0,0 @@
-DROP TABLE IF EXISTS gb_postcode;
-CREATE TABLE gb_postcode (
-    id integer,
-    postcode character varying(9),
-    geometry geometry,
-    CONSTRAINT enforce_dims_geometry CHECK ((st_ndims(geometry) = 2)),
-    CONSTRAINT enforce_srid_geometry CHECK ((st_srid(geometry) = 4326))
-);
-
-DROP TABLE IF EXISTS us_postcode;
-CREATE TABLE us_postcode (
-    postcode text,
-    x double precision,
-    y double precision
-);

lib-sql/tiger_import_finish.sql

@@ -1,17 +0,0 @@
---index only on parent_place_id
-CREATE INDEX IF NOT EXISTS idx_location_property_tiger_parent_place_id_imp
-  ON location_property_tiger_import (parent_place_id) {{db.tablespace.aux_index}};
-CREATE UNIQUE INDEX IF NOT EXISTS idx_location_property_tiger_place_id_imp
-  ON location_property_tiger_import (place_id) {{db.tablespace.aux_index}};
-
-GRANT SELECT ON location_property_tiger_import TO "{{config.DATABASE_WEBUSER}}";
-
-DROP TABLE IF EXISTS location_property_tiger;
-ALTER TABLE location_property_tiger_import RENAME TO location_property_tiger;
-
-ALTER INDEX IF EXISTS idx_location_property_tiger_parent_place_id_imp RENAME TO idx_location_property_tiger_housenumber_parent_place_id;
-ALTER INDEX IF EXISTS idx_location_property_tiger_place_id_imp RENAME TO idx_location_property_tiger_place_id;
-
-DROP FUNCTION tiger_line_import (linegeo GEOMETRY, in_startnumber INTEGER,
-                                 in_endnumber INTEGER, interpolationtype TEXT,
-                                 token_info JSONB, in_postcode TEXT);

lib-sql/tokenizer/icu_tokenizer_tables.sql

@@ -1,29 +0,0 @@
-DROP TABLE IF EXISTS word;
-CREATE TABLE word (
-  word_id INTEGER,
-  word_token text NOT NULL,
-  type text NOT NULL,
-  word text,
-  info jsonb
-) {{db.tablespace.search_data}};
-
-CREATE INDEX idx_word_word_token ON word
-    USING BTREE (word_token) {{db.tablespace.search_index}};
--- Used when updating country names from the boundary relation.
-CREATE INDEX idx_word_country_names ON word
-    USING btree(word) {{db.tablespace.address_index}}
-    WHERE type = 'C';
--- Used when inserting new postcodes on updates.
-CREATE INDEX idx_word_postcodes ON word
-    USING btree(word) {{db.tablespace.address_index}}
-    WHERE type = 'P';
--- Used when inserting full words.
-CREATE INDEX idx_word_full_word ON word
-    USING btree(word) {{db.tablespace.address_index}}
-    WHERE type = 'W';
-
-GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}";
-
-DROP SEQUENCE IF EXISTS seq_word;
-CREATE SEQUENCE seq_word start 1;
-GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}";

lib-sql/tokenizer/legacy_icu_tokenizer.sql

@@ -1,157 +0,0 @@
--- Get tokens used for searching the given place.
---
--- These are the tokens that will be saved in the search_name table.
-CREATE OR REPLACE FUNCTION token_get_name_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Get tokens for matching the place name against others.
---
--- This should usually be restricted to full name tokens.
-CREATE OR REPLACE FUNCTION token_get_name_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber tokens applicable for the place.
-CREATE OR REPLACE FUNCTION token_get_housenumber_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'hnr_tokens')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber in the form that it can be matched during search.
-CREATE OR REPLACE FUNCTION token_normalized_housenumber(info JSONB)
-  RETURNS TEXT
-AS $$
-  SELECT info->>'hnr';
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_street_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'street')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_match')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_search')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-DROP TYPE IF EXISTS token_addresstoken CASCADE;
-CREATE TYPE token_addresstoken AS (
-  key TEXT,
-  match_tokens INT[],
-  search_tokens INT[]
-);
-
-CREATE OR REPLACE FUNCTION token_get_address_tokens(info JSONB)
-  RETURNS SETOF token_addresstoken
-AS $$
-  SELECT key, (value->>1)::int[] as match_tokens,
-         (value->>0)::int[] as search_tokens
-  FROM jsonb_each(info->'addr');
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_normalized_postcode(postcode TEXT)
-  RETURNS TEXT
-AS $$
-  SELECT CASE WHEN postcode SIMILAR TO '%(,|;)%' THEN NULL ELSE upper(trim(postcode))END;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return token info that should be saved permanently in the database.
-CREATE OR REPLACE FUNCTION token_strip_info(info JSONB)
-  RETURNS JSONB
-AS $$
-  SELECT NULL::JSONB;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
---------------- private functions ----------------------------------------------
-
-CREATE OR REPLACE FUNCTION getorcreate_full_word(norm_term TEXT, lookup_terms TEXT[],
-                                                 OUT full_token INT,
-                                                 OUT partial_tokens INT[])
-  AS $$
-DECLARE
-  partial_terms TEXT[] = '{}'::TEXT[];
-  term TEXT;
-  term_id INTEGER;
-  term_count INTEGER;
-BEGIN
-  SELECT min(word_id) INTO full_token
-    FROM word WHERE word = norm_term and type = 'W';
-
-  IF full_token IS NULL THEN
-    full_token := nextval('seq_word');
-    INSERT INTO word (word_id, word_token, type, word, info)
-      SELECT full_token, lookup_term, 'W', norm_term,
-             json_build_object('count', 0)
-        FROM unnest(lookup_terms) as lookup_term;
-  END IF;
-
-  FOR term IN SELECT unnest(string_to_array(unnest(lookup_terms), ' ')) LOOP
-    term := trim(term);
-    IF NOT (ARRAY[term] <@ partial_terms) THEN
-      partial_terms := partial_terms || term;
-    END IF;
-  END LOOP;
-
-  partial_tokens := '{}'::INT[];
-  FOR term IN SELECT unnest(partial_terms) LOOP
-    SELECT min(word_id), max(info->>'count') INTO term_id, term_count
-      FROM word WHERE word_token = term and type = 'w';
-
-    IF term_id IS NULL THEN
-      term_id := nextval('seq_word');
-      term_count := 0;
-      INSERT INTO word (word_id, word_token, type, info)
-        VALUES (term_id, term, 'w', json_build_object('count', term_count));
-    END IF;
-
-    IF term_count < {{ max_word_freq }} THEN
-      partial_tokens := array_merge(partial_tokens, ARRAY[term_id]);
-    END IF;
-  END LOOP;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  return_id INTEGER;
-BEGIN
-  SELECT min(word_id) INTO return_id FROM word
-    WHERE word_token = lookup_term and type = 'H';
-
-  IF return_id IS NULL THEN
-    return_id := nextval('seq_word');
-    INSERT INTO word (word_id, word_token, type)
-      VALUES (return_id, lookup_term, 'H');
-  END IF;
-
-  RETURN return_id;
-END;
-$$
-LANGUAGE plpgsql;

lib-sql/tokenizer/legacy_tokenizer.sql

@@ -1,393 +0,0 @@
--- Get tokens used for searching the given place.
---
--- These are the tokens that will be saved in the search_name table.
-CREATE OR REPLACE FUNCTION token_get_name_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Get tokens for matching the place name against others.
---
--- This should usually be restricted to full name tokens.
-CREATE OR REPLACE FUNCTION token_get_name_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber tokens applicable for the place.
-CREATE OR REPLACE FUNCTION token_get_housenumber_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'hnr_tokens')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber in the form that it can be matched during search.
-CREATE OR REPLACE FUNCTION token_normalized_housenumber(info JSONB)
-  RETURNS TEXT
-AS $$
-  SELECT info->>'hnr';
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_street_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'street')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_match')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_search')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-DROP TYPE IF EXISTS token_addresstoken CASCADE;
-CREATE TYPE token_addresstoken AS (
-  key TEXT,
-  match_tokens INT[],
-  search_tokens INT[]
-);
-
-CREATE OR REPLACE FUNCTION token_get_address_tokens(info JSONB)
-  RETURNS SETOF token_addresstoken
-AS $$
-  SELECT key, (value->>1)::int[] as match_tokens,
-         (value->>0)::int[] as search_tokens
-  FROM jsonb_each(info->'addr');
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_normalized_postcode(postcode TEXT)
-  RETURNS TEXT
-AS $$
-  SELECT CASE WHEN postcode SIMILAR TO '%(,|;)%' THEN NULL ELSE upper(trim(postcode))END;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return token info that should be saved permanently in the database.
-CREATE OR REPLACE FUNCTION token_strip_info(info JSONB)
-  RETURNS JSONB
-AS $$
-  SELECT NULL::JSONB;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
---------------- private functions ----------------------------------------------
-
--- Functions for term normalisation and access to the 'word' table.
-
-CREATE OR REPLACE FUNCTION transliteration(text) RETURNS text
-  AS '{{ modulepath }}/nominatim.so', 'transliteration'
-LANGUAGE c IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION gettokenstring(text) RETURNS text
-  AS '{{ modulepath }}/nominatim.so', 'gettokenstring'
-LANGUAGE c IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION make_standard_name(name TEXT) RETURNS TEXT
-  AS $$
-DECLARE
-  o TEXT;
-BEGIN
-  o := public.gettokenstring(public.transliteration(name));
-  RETURN trim(substr(o,1,length(o)));
-END;
-$$
-LANGUAGE plpgsql IMMUTABLE;
-
--- returns NULL if the word is too common
-CREATE OR REPLACE FUNCTION getorcreate_word_id(lookup_word TEXT) 
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  return_word_id INTEGER;
-  count INTEGER;
-BEGIN
-  lookup_token := trim(lookup_word);
-  SELECT min(word_id), max(search_name_count) FROM word
-    WHERE word_token = lookup_token and class is null and type is null
-    INTO return_word_id, count;
-  IF return_word_id IS NULL THEN
-    return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, 0);
-  ELSE
-    IF count > {{ max_word_freq }} THEN
-      return_word_id := NULL;
-    END IF;
-  END IF;
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
--- Create housenumber tokens from an OSM addr:housenumber.
--- The housnumber is split at comma and semicolon as necessary.
--- The function returns the normalized form of the housenumber suitable
--- for comparison.
-CREATE OR REPLACE FUNCTION create_housenumbers(housenumbers TEXT[],
-                                               OUT tokens TEXT,
-                                               OUT normtext TEXT)
-  AS $$
-BEGIN
-  SELECT array_to_string(array_agg(trans), ';'), array_agg(tid)::TEXT
-    INTO normtext, tokens
-    FROM (SELECT lookup_word as trans, getorcreate_housenumber_id(lookup_word) as tid
-          FROM (SELECT make_standard_name(h) as lookup_word
-                FROM unnest(housenumbers) h) x) y;
-END;
-$$ LANGUAGE plpgsql STABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  return_word_id INTEGER;
-BEGIN
-  lookup_token := ' ' || trim(lookup_word);
-  SELECT min(word_id) FROM word
-    WHERE word_token = lookup_token and class='place' and type='house'
-    INTO return_word_id;
-  IF return_word_id IS NULL THEN
-    return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null,
-                             'place', 'house', null, 0);
-  END IF;
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION create_postcode_id(postcode TEXT)
-  RETURNS BOOLEAN
-  AS $$
-DECLARE
-  r RECORD;
-  lookup_token TEXT;
-  return_word_id INTEGER;
-BEGIN
-  lookup_token := ' ' || make_standard_name(postcode);
-  FOR r IN
-    SELECT word_id FROM word
-    WHERE word_token = lookup_token and word = postcode
-          and class='place' and type='postcode'
-  LOOP
-    RETURN false;
-  END LOOP;
-
-  INSERT INTO word VALUES (nextval('seq_word'), lookup_token, postcode,
-                           'place', 'postcode', null, 0);
-  RETURN true;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION getorcreate_name_id(lookup_word TEXT, src_word TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  nospace_lookup_token TEXT;
-  return_word_id INTEGER;
-BEGIN
-  lookup_token := ' '||trim(lookup_word);
-  SELECT min(word_id) FROM word
-  WHERE word_token = lookup_token and class is null and type is null
-  INTO return_word_id;
-  IF return_word_id IS NULL THEN
-    return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, src_word,
-                             null, null, null, 0);
-  END IF;
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
--- Normalize a string and lookup its word ids (partial words).
-CREATE OR REPLACE FUNCTION addr_ids_from_name(lookup_word TEXT)
-  RETURNS INTEGER[]
-  AS $$
-DECLARE
-  words TEXT[];
-  id INTEGER;
-  return_word_id INTEGER[];
-  word_ids INTEGER[];
-  j INTEGER;
-BEGIN
-  words := string_to_array(make_standard_name(lookup_word), ' ');
-  IF array_upper(words, 1) IS NOT NULL THEN
-    FOR j IN 1..array_upper(words, 1) LOOP
-      IF (words[j] != '') THEN
-        SELECT array_agg(word_id) INTO word_ids
-          FROM word
-         WHERE word_token = words[j] and class is null and type is null;
-
-        IF word_ids IS NULL THEN
-          id := nextval('seq_word');
-          INSERT INTO word VALUES (id, words[j], null, null, null, null, 0);
-          return_word_id := return_word_id || id;
-        ELSE
-          return_word_id := array_merge(return_word_id, word_ids);
-        END IF;
-      END IF;
-    END LOOP;
-  END IF;
-
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
--- Normalize a string and look up its name ids (full words).
-CREATE OR REPLACE FUNCTION word_ids_from_name(lookup_word TEXT)
-  RETURNS INTEGER[]
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  return_word_ids INTEGER[];
-BEGIN
-  lookup_token := ' '|| make_standard_name(lookup_word);
-  SELECT array_agg(word_id) FROM word
-    WHERE word_token = lookup_token and class is null and type is null
-    INTO return_word_ids;
-  RETURN return_word_ids;
-END;
-$$
-LANGUAGE plpgsql STABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION make_keywords(src HSTORE)
-  RETURNS INTEGER[]
-  AS $$
-DECLARE
-  result INTEGER[];
-  s TEXT;
-  w INTEGER;
-  words TEXT[];
-  value TEXT;
-  j INTEGER;
-BEGIN
-  result := '{}'::INTEGER[];
-
-  FOR value IN SELECT unnest(regexp_split_to_array(svals(src), E'[,;]')) LOOP
-    -- full name
-    s := make_standard_name(value);
-    w := getorcreate_name_id(s, value);
-
-    IF not(ARRAY[w] <@ result) THEN
-      result := result || w;
-    END IF;
-
-    -- partial single-word terms
-    words := string_to_array(s, ' ');
-    IF array_upper(words, 1) IS NOT NULL THEN
-      FOR j IN 1..array_upper(words, 1) LOOP
-        IF (words[j] != '') THEN
-          w = getorcreate_word_id(words[j]);
-          IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-            result := result || w;
-          END IF;
-        END IF;
-      END LOOP;
-    END IF;
-
-    -- consider parts before an opening braket a full word as well
-    words := regexp_split_to_array(value, E'[(]');
-    IF array_upper(words, 1) > 1 THEN
-      s := make_standard_name(words[1]);
-      IF s != '' THEN
-        w := getorcreate_name_id(s, words[1]);
-        IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-          result := result || w;
-        END IF;
-      END IF;
-    END IF;
-
-    s := regexp_replace(value, '市$', '');
-    IF s != value THEN
-      s := make_standard_name(s);
-      IF s != '' THEN
-        w := getorcreate_name_id(s, value);
-        IF NOT (ARRAY[w] <@ result) THEN
-          result := result || w;
-        END IF;
-      END IF;
-    END IF;
-
-  END LOOP;
-
-  RETURN result;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION precompute_words(src TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  s TEXT;
-  w INTEGER;
-  words TEXT[];
-  i INTEGER;
-  j INTEGER;
-BEGIN
-  s := make_standard_name(src);
-  w := getorcreate_name_id(s, src);
-
-  w := getorcreate_word_id(s);
-
-  words := string_to_array(s, ' ');
-  IF array_upper(words, 1) IS NOT NULL THEN
-    FOR j IN 1..array_upper(words, 1) LOOP
-      IF (words[j] != '') THEN
-        w := getorcreate_word_id(words[j]);
-      END IF;
-    END LOOP;
-  END IF;
-
-  words := regexp_split_to_array(src, E'[,;()]');
-  IF array_upper(words, 1) != 1 THEN
-    FOR j IN 1..array_upper(words, 1) LOOP
-      s := make_standard_name(words[j]);
-      IF s != '' THEN
-        w := getorcreate_word_id(s);
-      END IF;
-    END LOOP;
-  END IF;
-
-  s := regexp_replace(src, '市$', '');
-  IF s != src THEN
-    s := make_standard_name(s);
-    IF s != '' THEN
-      w := getorcreate_name_id(s, src);
-    END IF;
-  END IF;
-
-  RETURN 1;
-END;
-$$
-LANGUAGE plpgsql;

lib-sql/tokenizer/legacy_tokenizer_indices.sql

@@ -1,2 +0,0 @@
-CREATE INDEX IF NOT EXISTS idx_word_word_id
-  ON word USING BTREE (word_id) {{db.tablespace.search_index}};

lib-sql/tokenizer/legacy_tokenizer_tables.sql

@@ -1,21 +0,0 @@
-DROP TABLE IF EXISTS word;
-CREATE TABLE word (
-  word_id INTEGER,
-  word_token text NOT NULL,
-  word text,
-  class text,
-  type text,
-  country_code varchar(2),
-  search_name_count INTEGER,
-  operator TEXT
-) {{db.tablespace.search_data}};
-
-CREATE INDEX idx_word_word_token ON word
-    USING BTREE (word_token) {{db.tablespace.search_index}};
-CREATE INDEX idx_word_word ON word
-    USING BTREE (word) {{db.tablespace.search_index}} WHERE word is not null;
-GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}";
-
-DROP SEQUENCE IF EXISTS seq_word;
-CREATE SEQUENCE seq_word start 1;
-GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}";

lib/AddressDetails.php

@@ -2,7 +2,7 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/ClassTypes.php');
+require_once(CONST_BasePath.'/lib/ClassTypes.php');
 
 /**
  * Detailed list of address parts for a single result
@@ -61,7 +61,7 @@ class AddressDetails
         return join(', ', $aParts);
     }
 
-    public function getAddressNames()
+    public function getAddressNames($sCountry = null)
     {
         $aAddress = array();
 
@@ -79,11 +79,13 @@ class AddressDetails
                 $sName = $aLine['housenumber'];
             }
 
-            if (isset($sName)
-                && (!isset($aAddress[$sTypeLabel])
-                    || $aLine['class'] == 'place')
-            ) {
-                $aAddress[$sTypeLabel] = $sName;
+            if (isset($sName)) {
+                $sTypeLabel = strtolower(str_replace(' ', '_', $sTypeLabel));
+                if (!isset($aAddress[$sTypeLabel])
+                    || $aLine['class'] == 'place'
+                ) {
+                    $aAddress[$sTypeLabel] = $sName;
+                }
             }
         }
 

lib/DB.php

@@ -2,7 +2,7 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/DatabaseError.php');
+require_once(CONST_BasePath.'/lib/DatabaseError.php');
 
 /**
  * Uses PDO to access the database specified in the CONST_Database_DSN
@@ -12,9 +12,9 @@ class DB
 {
     protected $connection;
 
-    public function __construct($sDSN = null)
+    public function __construct($sDSN = CONST_Database_DSN)
     {
-        $this->sDSN = $sDSN ?? getSetting('DATABASE_DSN');
+        $this->sDSN = $sDSN;
     }
 
     public function connect($bNew = false, $bPersistent = true)
@@ -39,9 +39,7 @@ class DB
         $conn->exec("SET DateStyle TO 'sql,european'");
         $conn->exec("SET client_encoding TO 'utf-8'");
         $iMaxExecution = ini_get('max_execution_time');
-        if ($iMaxExecution > 0) {
-            $conn->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds
-        }
+        if ($iMaxExecution > 0) $conn->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds
 
         $this->connection = $conn;
         return true;
@@ -97,9 +95,7 @@ class DB
         try {
             $stmt = $this->getQueryStatement($sSQL, $aInputVars, $sErrMessage);
             $row = $stmt->fetch(\PDO::FETCH_NUM);
-            if ($row === false) {
-                return false;
-            }
+            if ($row === false) return false;
         } catch (\PDOException $e) {
             throw new \Nominatim\DatabaseError($sErrMessage, 500, null, $e, $sSQL);
         }
@@ -244,6 +240,16 @@ class DB
         return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1);
     }
 
+    /**
+    * Returns a list of table names in the database
+    *
+    * @return array[]
+    */
+    public function getListOfTables()
+    {
+        return $this->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
+    }
+
     /**
      * Deletes a table. Returns true if deleted or didn't exist.
      *
@@ -256,6 +262,76 @@ class DB
         return $this->exec('DROP TABLE IF EXISTS '.$sTableName.' CASCADE') == 0;
     }
 
+    /**
+    * Check if an index exists in the database. Optional filtered by tablename
+    *
+    * @param string  $sTableName
+    *
+    * @return boolean
+    */
+    public function indexExists($sIndexName, $sTableName = null)
+    {
+        return in_array($sIndexName, $this->getListOfIndices($sTableName));
+    }
+
+    /**
+    * Returns a list of index names in the database, optional filtered by tablename
+    *
+    * @param string  $sTableName
+    *
+    * @return array
+    */
+    public function getListOfIndices($sTableName = null)
+    {
+        //  table_name            | index_name                      | column_name
+        // -----------------------+---------------------------------+--------------
+        //  country_name          | idx_country_name_country_code   | country_code
+        //  country_osm_grid      | idx_country_osm_grid_geometry   | geometry
+        //  import_polygon_delete | idx_import_polygon_delete_osmid | osm_id
+        //  import_polygon_delete | idx_import_polygon_delete_osmid | osm_type
+        //  import_polygon_error  | idx_import_polygon_error_osmid  | osm_id
+        //  import_polygon_error  | idx_import_polygon_error_osmid  | osm_type
+        $sSql = <<< END
+SELECT
+    t.relname as table_name,
+    i.relname as index_name,
+    a.attname as column_name
+FROM
+    pg_class t,
+    pg_class i,
+    pg_index ix,
+    pg_attribute a
+WHERE
+    t.oid = ix.indrelid
+    and i.oid = ix.indexrelid
+    and a.attrelid = t.oid
+    and a.attnum = ANY(ix.indkey)
+    and t.relkind = 'r'
+    and i.relname NOT LIKE 'pg_%'
+    FILTERS
+ ORDER BY
+    t.relname,
+    i.relname,
+    a.attname
+END;
+
+        $aRows = null;
+        if ($sTableName) {
+            $sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql);
+            $aRows = $this->getAll($sSql, array(':tablename' => $sTableName));
+        } else {
+            $sSql = str_replace('FILTERS', '', $sSql);
+            $aRows = $this->getAll($sSql);
+        }
+
+        $aIndexNames = array_unique(array_map(function ($aRow) {
+            return $aRow['index_name'];
+        }, $aRows));
+        sort($aIndexNames);
+
+        return $aIndexNames;
+    }
+
     /**
      * Tries to connect to the database but on failure doesn't throw an exception.
      *
@@ -310,13 +386,9 @@ class DB
         if (preg_match('/^pgsql:(.+)$/', $sDSN, $aMatches)) {
             foreach (explode(';', $aMatches[1]) as $sKeyVal) {
                 list($sKey, $sVal) = explode('=', $sKeyVal, 2);
-                if ($sKey == 'host') {
-                    $sKey = 'hostspec';
-                } elseif ($sKey == 'dbname') {
-                    $sKey = 'database';
-                } elseif ($sKey == 'user') {
-                    $sKey = 'username';
-                }
+                if ($sKey == 'host') $sKey = 'hostspec';
+                if ($sKey == 'dbname') $sKey = 'database';
+                if ($sKey == 'user') $sKey = 'username';
                 $aInfo[$sKey] = $sVal;
             }
         }

lib/DatabaseError.php

@@ -5,7 +5,7 @@ namespace Nominatim;
 class DatabaseError extends \Exception
 {
 
-    public function __construct($message, $code, $previous, $oPDOErr, $sSql = null)
+    public function __construct($message, $code = 500, Exception $previous = null, $oPDOErr, $sSql = null)
     {
         parent::__construct($message, $code, $previous);
         // https://secure.php.net/manual/en/class.pdoexception.php

lib/DebugHtml.php

@@ -78,7 +78,7 @@ class Debug
         echo '<th>Address Tokens</th><th>Address Not</th>';
         echo '<th>country</th><th>operator</th>';
         echo '<th>class</th><th>type</th><th>postcode</th><th>housenumber</th></tr>';
-        foreach ($aSearches as $aRankedSet) {
+        foreach ($aSearches as $iRank => $aRankedSet) {
             foreach ($aRankedSet as $aRow) {
                 $aRow->dumpAsHtmlTableRow($aWordsIDs);
             }

lib/Geocode.php

@@ -2,25 +2,23 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/PlaceLookup.php');
-require_once(CONST_LibDir.'/Phrase.php');
-require_once(CONST_LibDir.'/ReverseGeocode.php');
-require_once(CONST_LibDir.'/SearchDescription.php');
-require_once(CONST_LibDir.'/SearchContext.php');
-require_once(CONST_LibDir.'/SearchPosition.php');
-require_once(CONST_LibDir.'/TokenList.php');
-require_once(CONST_TokenizerDir.'/tokenizer.php');
+require_once(CONST_BasePath.'/lib/PlaceLookup.php');
+require_once(CONST_BasePath.'/lib/Phrase.php');
+require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
+require_once(CONST_BasePath.'/lib/SearchDescription.php');
+require_once(CONST_BasePath.'/lib/SearchContext.php');
+require_once(CONST_BasePath.'/lib/TokenList.php');
 
 class Geocode
 {
     protected $oDB;
 
     protected $oPlaceLookup;
-    protected $oTokenizer;
 
     protected $aLangPrefOrder = array();
 
     protected $aExcludePlaceIDs = array();
+    protected $bReverseInPlan = false;
 
     protected $iLimit = 20;
     protected $iFinalLimit = 10;
@@ -44,12 +42,28 @@ class Geocode
     protected $sQuery = false;
     protected $aStructuredQuery = false;
 
+    protected $oNormalizer = null;
+
 
     public function __construct(&$oDB)
     {
         $this->oDB =& $oDB;
         $this->oPlaceLookup = new PlaceLookup($this->oDB);
-        $this->oTokenizer = new \Nominatim\Tokenizer($this->oDB);
+        $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
+    }
+
+    private function normTerm($sTerm)
+    {
+        if ($this->oNormalizer === null) {
+            return $sTerm;
+        }
+
+        return $this->oNormalizer->transliterate($sTerm);
+    }
+
+    public function setReverseInPlan($bReverse)
+    {
+        $this->bReverseInPlan = $bReverse;
     }
 
     public function setLanguagePreference($aLangPref)
@@ -71,9 +85,7 @@ class Geocode
             $aParams['exclude_place_ids'] = implode(',', $this->aExcludePlaceIDs);
         }
 
-        if ($this->bBoundedSearch) {
-            $aParams['bounded'] = '1';
-        }
+        if ($this->bBoundedSearch) $aParams['bounded'] = '1';
 
         if ($this->aCountryCodes) {
             $aParams['countrycodes'] = implode(',', $this->aCountryCodes);
@@ -88,11 +100,8 @@ class Geocode
 
     public function setLimit($iLimit = 10)
     {
-        if ($iLimit > 50) {
-            $iLimit = 50;
-        } elseif ($iLimit < 1) {
-            $iLimit = 1;
-        }
+        if ($iLimit > 50) $iLimit = 50;
+        if ($iLimit < 1) $iLimit = 1;
 
         $this->iFinalLimit = $iLimit;
         $this->iLimit = $iLimit + min($iLimit, 10);
@@ -187,24 +196,18 @@ class Geocode
         if ($sExcluded) {
             foreach ($sExcluded as $iExcludedPlaceID) {
                 $iExcludedPlaceID = (int)$iExcludedPlaceID;
-                if ($iExcludedPlaceID) {
+                if ($iExcludedPlaceID)
                     $aExcludePlaceIDs[$iExcludedPlaceID] = $iExcludedPlaceID;
-                }
             }
 
-            if (isset($aExcludePlaceIDs)) {
+            if (isset($aExcludePlaceIDs))
                 $this->aExcludePlaceIDs = $aExcludePlaceIDs;
-            }
         }
 
         // Only certain ranks of feature
         $sFeatureType = $oParams->getString('featureType');
-        if (!$sFeatureType) {
-            $sFeatureType = $oParams->getString('featuretype');
-        }
-        if ($sFeatureType) {
-            $this->setFeatureType($sFeatureType);
-        }
+        if (!$sFeatureType) $sFeatureType = $oParams->getString('featuretype');
+        if ($sFeatureType) $this->setFeatureType($sFeatureType);
 
         // Country code list
         $sCountries = $oParams->getStringList('countrycodes');
@@ -214,9 +217,8 @@ class Geocode
                     $aCountries[] = strtolower($sCountryCode);
                 }
             }
-            if (isset($aCountries)) {
+            if (isset($aCountries))
                 $this->aCountryCodes = $aCountries;
-            }
         }
 
         $aViewbox = $oParams->getStringList('viewboxlbrt');
@@ -260,6 +262,7 @@ class Geocode
                 $oParams->getString('country'),
                 $oParams->getString('postalcode')
             );
+            $this->setReverseInPlan(false);
         } else {
             $this->setQuery($sQuery);
         }
@@ -268,17 +271,13 @@ class Geocode
     public function loadStructuredAddressElement($sValue, $sKey, $iNewMinAddressRank, $iNewMaxAddressRank, $aItemListValues)
     {
         $sValue = trim($sValue);
-        if (!$sValue) {
-            return false;
-        }
+        if (!$sValue) return false;
         $this->aStructuredQuery[$sKey] = $sValue;
         if ($this->iMinAddressRank == 0 && $this->iMaxAddressRank == 30) {
             $this->iMinAddressRank = $iNewMinAddressRank;
             $this->iMaxAddressRank = $iNewMaxAddressRank;
         }
-        if ($aItemListValues) {
-            $this->aAddressRankList = array_merge($this->aAddressRankList, $aItemListValues);
-        }
+        if ($aItemListValues) $this->aAddressRankList = array_merge($this->aAddressRankList, $aItemListValues);
         return true;
     }
 
@@ -312,11 +311,11 @@ class Geocode
 
     public function fallbackStructuredQuery()
     {
+        if (!$this->aStructuredQuery) return false;
+
         $aParams = $this->aStructuredQuery;
 
-        if (!$aParams || count($aParams) == 1) {
-            return false;
-        }
+        if (count($aParams) == 1) return false;
 
         $aOrderToFallback = array('postalcode', 'street', 'city', 'county', 'state');
 
@@ -331,7 +330,7 @@ class Geocode
         return false;
     }
 
-    public function getGroupedSearches($aSearches, $aPhrases, $oValidTokens)
+    public function getGroupedSearches($aSearches, $aPhrases, $oValidTokens, $bIsStructured)
     {
         /*
              Calculate all searches using oValidTokens i.e.
@@ -346,26 +345,52 @@ class Geocode
          */
         foreach ($aPhrases as $iPhrase => $oPhrase) {
             $aNewPhraseSearches = array();
-            $oPosition = new SearchPosition(
-                $oPhrase->getPhraseType(),
-                $iPhrase,
-                count($aPhrases)
-            );
+            $sPhraseType = $bIsStructured ? $oPhrase->getPhraseType() : '';
 
             foreach ($oPhrase->getWordSets() as $aWordset) {
                 $aWordsetSearches = $aSearches;
 
                 // Add all words from this wordset
                 foreach ($aWordset as $iToken => $sToken) {
+                    //echo "<br><b>$sToken</b>";
                     $aNewWordsetSearches = array();
-                    $oPosition->setTokenPosition($iToken, count($aWordset));
 
                     foreach ($aWordsetSearches as $oCurrentSearch) {
-                        foreach ($oValidTokens->get($sToken) as $oSearchTerm) {
-                            if ($oSearchTerm->isExtendable($oCurrentSearch, $oPosition)) {
-                                $aNewSearches = $oSearchTerm->extendSearch(
-                                    $oCurrentSearch,
-                                    $oPosition
+                        //echo "<i>";
+                        //var_dump($oCurrentSearch);
+                        //echo "</i>";
+
+                        // Tokens with full name matches.
+                        foreach ($oValidTokens->get(' '.$sToken) as $oSearchTerm) {
+                            $aNewSearches = $oCurrentSearch->extendWithFullTerm(
+                                $oSearchTerm,
+                                $oValidTokens->contains($sToken)
+                                  && strpos($sToken, ' ') === false,
+                                $sPhraseType,
+                                $iToken == 0 && $iPhrase == 0,
+                                $iPhrase == 0,
+                                $iToken + 1 == count($aWordset)
+                                  && $iPhrase + 1 == count($aPhrases)
+                            );
+
+                            foreach ($aNewSearches as $oSearch) {
+                                if ($oSearch->getRank() < $this->iMaxRank) {
+                                    $aNewWordsetSearches[] = $oSearch;
+                                }
+                            }
+                        }
+                        // Look for partial matches.
+                        // Note that there is no point in adding country terms here
+                        // because country is omitted in the address.
+                        if ($sPhraseType != 'country') {
+                            // Allow searching for a word - but at extra cost
+                            foreach ($oValidTokens->get($sToken) as $oSearchTerm) {
+                                $aNewSearches = $oCurrentSearch->extendWithPartialTerm(
+                                    $sToken,
+                                    $oSearchTerm,
+                                    $bIsStructured,
+                                    $iPhrase,
+                                    $oValidTokens->get(' '.$sToken)
                                 );
 
                                 foreach ($aNewSearches as $oSearch) {
@@ -380,6 +405,7 @@ class Geocode
                     usort($aNewWordsetSearches, array('Nominatim\SearchDescription', 'bySearchRank'));
                     $aWordsetSearches = array_slice($aNewWordsetSearches, 0, 50);
                 }
+                //var_Dump('<hr>',count($aWordsetSearches)); exit;
 
                 $aNewPhraseSearches = array_merge($aNewPhraseSearches, $aNewWordsetSearches);
                 usort($aNewPhraseSearches, array('Nominatim\SearchDescription', 'bySearchRank'));
@@ -387,11 +413,8 @@ class Geocode
                 $aSearchHash = array();
                 foreach ($aNewPhraseSearches as $iSearch => $aSearch) {
                     $sHash = serialize($aSearch);
-                    if (isset($aSearchHash[$sHash])) {
-                        unset($aNewPhraseSearches[$iSearch]);
-                    } else {
-                        $aSearchHash[$sHash] = 1;
-                    }
+                    if (isset($aSearchHash[$sHash])) unset($aNewPhraseSearches[$iSearch]);
+                    else $aSearchHash[$sHash] = 1;
                 }
 
                 $aNewPhraseSearches = array_slice($aNewPhraseSearches, 0, 50);
@@ -412,12 +435,10 @@ class Geocode
 
             $iSearchCount = 0;
             $aSearches = array();
-            foreach ($aGroupedSearches as $aNewSearches) {
+            foreach ($aGroupedSearches as $iScore => $aNewSearches) {
                 $iSearchCount += count($aNewSearches);
                 $aSearches = array_merge($aSearches, $aNewSearches);
-                if ($iSearchCount > 50) {
-                    break;
-                }
+                if ($iSearchCount > 50) break;
             }
         }
 
@@ -474,9 +495,7 @@ class Geocode
     public function lookup()
     {
         Debug::newFunction('Geocode::lookup');
-        if (!$this->sQuery && !$this->aStructuredQuery) {
-            return array();
-        }
+        if (!$this->sQuery && !$this->aStructuredQuery) return array();
 
         Debug::printDebugArray('Geocode', $this);
 
@@ -498,10 +517,16 @@ class Geocode
         if ($this->aCountryCodes) {
             $oCtx->setCountryList($this->aCountryCodes);
         }
-        $this->oTokenizer->setCountryRestriction($this->aCountryCodes);
 
         Debug::newSection('Query Preprocessing');
 
+        $sNormQuery = $this->normTerm($this->sQuery);
+        Debug::printVar('Normalized query', $sNormQuery);
+
+        $sLanguagePrefArraySQL = $this->oDB->getArraySQL(
+            $this->oDB->getDBQuotedList($this->aLangPrefOrder)
+        );
+
         $sQuery = $this->sQuery;
         if (!preg_match('//u', $sQuery)) {
             userError('Query string is not UTF-8 encoded.');
@@ -551,62 +576,117 @@ class Geocode
             }
 
             if ($sSpecialTerm && !$aSearches[0]->hasOperator()) {
-                $aTokens = $this->oTokenizer->tokensForSpecialTerm($sSpecialTerm);
+                $sSpecialTerm = pg_escape_string($sSpecialTerm);
+                $sToken = $this->oDB->getOne(
+                    'SELECT make_standard_name(:term)',
+                    array(':term' => $sSpecialTerm),
+                    'Cannot decode query. Wrong encoding?'
+                );
+                $sSQL = 'SELECT class, type FROM word ';
+                $sSQL .= '   WHERE word_token in (\' '.$sToken.'\')';
+                $sSQL .= '   AND class is not null AND class not in (\'place\')';
 
-                if (!empty($aTokens)) {
-                    $aNewSearches = array();
-                    $oPosition = new SearchPosition('', 0, 1);
-                    $oPosition->setTokenPosition(0, 1);
-
-                    foreach ($aSearches as $oSearch) {
-                        foreach ($aTokens as $oToken) {
-                            $aNewSearches = array_merge(
-                                $aNewSearches,
-                                $oToken->extendSearch($oSearch, $oPosition)
-                            );
-                        }
+                Debug::printSQL($sSQL);
+                $aSearchWords = $this->oDB->getAll($sSQL);
+                $aNewSearches = array();
+                foreach ($aSearches as $oSearch) {
+                    foreach ($aSearchWords as $aSearchTerm) {
+                        $oNewSearch = clone $oSearch;
+                        $oNewSearch->setPoiSearch(
+                            Operator::TYPE,
+                            $aSearchTerm['class'],
+                            $aSearchTerm['type']
+                        );
+                        $aNewSearches[] = $oNewSearch;
                     }
-                    $aSearches = $aNewSearches;
                 }
+                $aSearches = $aNewSearches;
             }
 
             // Split query into phrases
             // Commas are used to reduce the search space by indicating where phrases split
-            $aPhrases = array();
             if ($this->aStructuredQuery) {
-                foreach ($this->aStructuredQuery as $iPhrase => $sPhrase) {
-                    $aPhrases[] = new Phrase($sPhrase, $iPhrase);
-                }
+                $aInPhrases = $this->aStructuredQuery;
+                $bStructuredPhrases = true;
             } else {
-                foreach (explode(',', $sQuery) as $sPhrase) {
-                    $aPhrases[] = new Phrase($sPhrase, '');
-                }
+                $aInPhrases = explode(',', $sQuery);
+                $bStructuredPhrases = false;
             }
 
             Debug::printDebugArray('Search context', $oCtx);
             Debug::printDebugArray('Base search', empty($aSearches) ? null : $aSearches[0]);
+            Debug::printVar('Final query phrases', $aInPhrases);
 
+            // Convert each phrase to standard form
+            // Create a list of standard words
+            // Get all 'sets' of words
+            // Generate a complete list of all
             Debug::newSection('Tokenization');
-            $oValidTokens = $this->oTokenizer->extractTokensFromPhrases($aPhrases);
+            $aTokens = array();
+            $aPhrases = array();
+            foreach ($aInPhrases as $iPhrase => $sPhrase) {
+                $sPhrase = $this->oDB->getOne(
+                    'SELECT make_standard_name(:phrase)',
+                    array(':phrase' => $sPhrase),
+                    'Cannot normalize query string (is it a UTF-8 string?)'
+                );
+                if (trim($sPhrase)) {
+                    $oPhrase = new Phrase($sPhrase, is_string($iPhrase) ? $iPhrase : '');
+                    $oPhrase->addTokens($aTokens);
+                    $aPhrases[] = $oPhrase;
+                }
+            }
+
+            Debug::printVar('Tokens', $aTokens);
+
+            $oValidTokens = new TokenList();
+
+            if (!empty($aTokens)) {
+                $oValidTokens->addTokensFromDB(
+                    $this->oDB,
+                    $aTokens,
+                    $this->aCountryCodes,
+                    $sNormQuery,
+                    $this->oNormalizer
+                );
 
-            if ($oValidTokens->count() > 0) {
                 $oCtx->setFullNameWords($oValidTokens->getFullWordIDs());
 
-                $aPhrases = array_filter($aPhrases, function ($oPhrase) {
-                    return $oPhrase->getWordSets() !== null;
-                });
+                // Try more interpretations for Tokens that could not be matched.
+                foreach ($aTokens as $sToken) {
+                    if ($sToken[0] == ' ' && !$oValidTokens->contains($sToken)) {
+                        if (preg_match('/^ ([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
+                            // US ZIP+4 codes - merge in the 5-digit ZIP code
+                            $oValidTokens->addToken(
+                                $sToken,
+                                new Token\Postcode(null, $aData[1], 'us')
+                            );
+                        } elseif (preg_match('/^ [0-9]+$/', $sToken)) {
+                            // Unknown single word token with a number.
+                            // Assume it is a house number.
+                            $oValidTokens->addToken(
+                                $sToken,
+                                new Token\HouseNumber(null, trim($sToken))
+                            );
+                        }
+                    }
+                }
 
                 // Any words that have failed completely?
                 // TODO: suggestions
 
                 Debug::printGroupTable('Valid Tokens', $oValidTokens->debugInfo());
+
+                foreach ($aPhrases as $oPhrase) {
+                    $oPhrase->computeWordSets($oValidTokens);
+                }
                 Debug::printDebugTable('Phrases', $aPhrases);
 
                 Debug::newSection('Search candidates');
 
-                $aGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens);
+                $aGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens, $bStructuredPhrases);
 
-                if (!$this->aStructuredQuery) {
+                if ($this->bReverseInPlan) {
                     // Reverse phrase array and also reverse the order of the wordsets in
                     // the first and final phrase. Don't bother about phrases in the middle
                     // because order in the address doesn't matter.
@@ -615,7 +695,7 @@ class Geocode
                     if (count($aPhrases) > 1) {
                         $aPhrases[count($aPhrases)-1]->invertWordSets();
                     }
-                    $aReverseGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens);
+                    $aReverseGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens, false);
 
                     foreach ($aGroupedSearches as $aSearches) {
                         foreach ($aSearches as $aSearch) {
@@ -634,9 +714,7 @@ class Geocode
                 $aGroupedSearches = array();
                 foreach ($aSearches as $aSearch) {
                     if ($aSearch->getRank() < $this->iMaxRank) {
-                        if (!isset($aGroupedSearches[$aSearch->getRank()])) {
-                            $aGroupedSearches[$aSearch->getRank()] = array();
-                        }
+                        if (!isset($aGroupedSearches[$aSearch->getRank()])) $aGroupedSearches[$aSearch->getRank()] = array();
                         $aGroupedSearches[$aSearch->getRank()][] = $aSearch;
                     }
                 }
@@ -650,9 +728,7 @@ class Geocode
                     $sHash = serialize($aSearch);
                     if (isset($aSearchHash[$sHash])) {
                         unset($aGroupedSearches[$iGroup][$iSearch]);
-                        if (empty($aGroupedSearches[$iGroup])) {
-                            unset($aGroupedSearches[$iGroup]);
-                        }
+                        if (empty($aGroupedSearches[$iGroup])) unset($aGroupedSearches[$iGroup]);
                     } else {
                         $aSearchHash[$sHash] = 1;
                     }
@@ -696,27 +772,20 @@ class Geocode
                         }
                     }
 
-                    if ($iQueryLoop > 20) {
-                        break;
-                    }
+                    if ($iQueryLoop > 20) break;
                 }
 
                 if (!empty($aResults)) {
                     $aSplitResults = Result::splitResults($aResults);
                     Debug::printVar('Split results', $aSplitResults);
-                    if ($iGroupLoop <= 4
-                        && reset($aSplitResults['head'])->iResultRank > 0
-                        && $iGroupedRank !== array_key_last($aGroupedSearches)) {
+                    if ($iGroupLoop <= 4 && empty($aSplitResults['tail'])
+                        && reset($aSplitResults['head'])->iResultRank > 0) {
                         // Haven't found an exact match for the query yet.
                         // Therefore add result from the next group level.
                         $aNextResults = $aSplitResults['head'];
                         foreach ($aNextResults as $oRes) {
                             $oRes->iResultRank--;
                         }
-                        foreach ($aSplitResults['tail'] as $oRes) {
-                            $oRes->iResultRank--;
-                            $aNextResults[$oRes->iId] = $oRes;
-                        }
                         $aResults = array();
                     } else {
                         $aResults = $aSplitResults['head'];
@@ -764,6 +833,7 @@ class Geocode
                     foreach ($aResults as $oResult) {
                         if (($this->iMaxAddressRank == 30 &&
                              ($oResult->iTable == Result::TABLE_OSMLINE
+                              || $oResult->iTable == Result::TABLE_AUX
                               || $oResult->iTable == Result::TABLE_TIGER))
                             || in_array($oResult->iId, $aFilteredIDs)
                         ) {
@@ -773,9 +843,9 @@ class Geocode
                     $aResults = $tempIDs;
                 }
 
-                if (!empty($aResults) || $iGroupLoop > 4 || $iQueryLoop > 30) {
-                    break;
-                }
+                if (!empty($aResults)) break;
+                if ($iGroupLoop > 4) break;
+                if ($iQueryLoop > 30) break;
             }
         } else {
             // Just interpret as a reverse geocode
@@ -793,8 +863,10 @@ class Geocode
 
         // No results? Done
         if (empty($aResults)) {
-            if ($this->bFallback && $this->fallbackStructuredQuery()) {
-                return $this->lookup();
+            if ($this->bFallback) {
+                if ($this->fallbackStructuredQuery()) {
+                    return $this->lookup();
+                }
             }
 
             return array();
@@ -813,9 +885,7 @@ class Geocode
 
         $aRecheckWords = preg_split('/\b[\s,\\-]*/u', $sQuery);
         foreach ($aRecheckWords as $i => $sWord) {
-            if (!preg_match('/[\pL\pN]/', $sWord)) {
-                unset($aRecheckWords[$i]);
-            }
+            if (!preg_match('/[\pL\pN]/', $sWord)) unset($aRecheckWords[$i]);
         }
 
         Debug::printVar('Recheck words', $aRecheckWords);
@@ -875,9 +945,7 @@ class Geocode
                 foreach ($aRecheckWords as $i => $sWord) {
                     if (stripos($sAddress, $sWord)!==false) {
                         $iCountWords++;
-                        if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) {
-                            $iCountWords += 0.1;
-                        }
+                        if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) $iCountWords += 0.1;
                     }
                 }
 
@@ -894,8 +962,15 @@ class Geocode
         $aToFilter = $aSearchResults;
         $aSearchResults = array();
 
+        $bFirst = true;
         foreach ($aToFilter as $aResult) {
             $this->aExcludePlaceIDs[$aResult['place_id']] = $aResult['place_id'];
+            if ($bFirst) {
+                $fLat = $aResult['lat'];
+                $fLon = $aResult['lon'];
+                if (isset($aResult['zoom'])) $iZoom = $aResult['zoom'];
+                $bFirst = false;
+            }
             if (!$this->oPlaceLookup->doDeDupe() || (!isset($aOSMIDDone[$aResult['osm_type'].$aResult['osm_id']])
                 && !isset($aClassTypeNameDone[$aResult['osm_type'].$aResult['class'].$aResult['type'].$aResult['name'].$aResult['admin_level']]))
             ) {
@@ -905,9 +980,7 @@ class Geocode
             }
 
             // Absolute limit on number of results
-            if (count($aSearchResults) >= $this->iFinalLimit) {
-                break;
-            }
+            if (count($aSearchResults) >= $this->iFinalLimit) break;
         }
 
         Debug::printVar('Post-filter results', $aSearchResults);
@@ -921,6 +994,7 @@ class Geocode
                 'Structured query' => $this->aStructuredQuery,
                 'Name keys' => Debug::fmtArrayVals($this->aLangPrefOrder),
                 'Excluded place IDs' => Debug::fmtArrayVals($this->aExcludePlaceIDs),
+                'Try reversed query'=> $this->bReverseInPlan,
                 'Limit (for searches)' => $this->iLimit,
                 'Limit (for results)'=> $this->iFinalLimit,
                 'Country codes' => Debug::fmtArrayVals($this->aCountryCodes),

lib/ParameterParser.php

@@ -90,16 +90,14 @@ class ParameterParser
         $aLanguages = array();
         $sLangString = $this->getString('accept-language', $sFallback);
 
-        if ($sLangString
-            && preg_match_all('/(([a-z]{1,8})([-_][a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $sLangString, $aLanguagesParse, PREG_SET_ORDER)
-        ) {
-            foreach ($aLanguagesParse as $iLang => $aLanguage) {
-                $aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
-                if (!isset($aLanguages[$aLanguage[2]])) {
-                    $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
+        if ($sLangString) {
+            if (preg_match_all('/(([a-z]{1,8})([-_][a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $sLangString, $aLanguagesParse, PREG_SET_ORDER)) {
+                foreach ($aLanguagesParse as $iLang => $aLanguage) {
+                    $aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
+                    if (!isset($aLanguages[$aLanguage[2]])) $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
                 }
+                arsort($aLanguages);
             }
-            arsort($aLanguages);
         }
         if (empty($aLanguages) && CONST_Default_Language) {
             $aLanguages[CONST_Default_Language] = 1;

lib/Phrase.php

@@ -16,6 +16,8 @@ class Phrase
     private $sPhrase;
     // Element type for structured searches.
     private $sPhraseType;
+    // Space-separated words of the phrase.
+    private $aWords;
     // Possible segmentations of the phrase.
     private $aWordSets;
 
@@ -36,14 +38,7 @@ class Phrase
     {
         $this->sPhrase = trim($sPhrase);
         $this->sPhraseType = $sPhraseType;
-    }
-
-    /**
-     * Get the orginal phrase of the string.
-     */
-    public function getPhrase()
-    {
-        return $this->sPhrase;
+        $this->aWords = explode(' ', $this->sPhrase);
     }
 
     /**
@@ -68,6 +63,30 @@ class Phrase
         return $this->aWordSets;
     }
 
+    /**
+     * Add the tokens from this phrase to the given list of tokens.
+     *
+     * @param string[] $aTokens List of tokens to append.
+     *
+     * @return void
+     */
+    public function addTokens(&$aTokens)
+    {
+        $iNumWords = count($this->aWords);
+
+        for ($i = 0; $i < $iNumWords; $i++) {
+            $sPhrase = $this->aWords[$i];
+            $aTokens[' '.$sPhrase] = ' '.$sPhrase;
+            $aTokens[$sPhrase] = $sPhrase;
+
+            for ($j = $i + 1; $j < $iNumWords; $j++) {
+                $sPhrase .= ' '.$this->aWords[$j];
+                $aTokens[' '.$sPhrase] = ' '.$sPhrase;
+                $aTokens[$sPhrase] = $sPhrase;
+            }
+        }
+    }
+
     /**
      * Invert the set of possible segmentations.
      *
@@ -80,27 +99,21 @@ class Phrase
         }
     }
 
-    public function computeWordSets($aWords, $oTokens)
+    public function computeWordSets($oTokens)
     {
-        $iNumWords = count($aWords);
-
-        if ($iNumWords == 0) {
-            $this->aWordSets = null;
-            return;
-        }
-
+        $iNumWords = count($this->aWords);
         // Caches the word set for the partial phrase up to word i.
         $aSetCache = array_fill(0, $iNumWords, array());
 
         // Initialise first element of cache. There can only be the word.
-        if ($oTokens->containsAny($aWords[0])) {
-            $aSetCache[0][] = array($aWords[0]);
+        if ($oTokens->containsAny($this->aWords[0])) {
+            $aSetCache[0][] = array($this->aWords[0]);
         }
 
         // Now do the next elements using what we already have.
         for ($i = 1; $i < $iNumWords; $i++) {
             for ($j = $i; $j > 0; $j--) {
-                $sPartial = $j == $i ? $aWords[$j] : $aWords[$j].' '.$sPartial;
+                $sPartial = $j == $i ? $this->aWords[$j] : $this->aWords[$j].' '.$sPartial;
                 if (!empty($aSetCache[$j - 1]) && $oTokens->containsAny($sPartial)) {
                     $aPartial = array($sPartial);
                     foreach ($aSetCache[$j - 1] as $aSet) {
@@ -123,7 +136,7 @@ class Phrase
             }
 
             // finally the current full phrase
-            $sPartial = $aWords[0].' '.$sPartial;
+            $sPartial = $this->aWords[0].' '.$sPartial;
             if ($oTokens->containsAny($sPartial)) {
                 $aSetCache[$i][] = array($sPartial);
             }
@@ -140,6 +153,7 @@ class Phrase
         return array(
                 'Type' => $this->sPhraseType,
                 'Phrase' => $this->sPhrase,
+                'Words' => $this->aWords,
                 'WordSets' => $this->aWordSets
                );
     }

lib/PlaceLookup.php

@@ -2,8 +2,8 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/AddressDetails.php');
-require_once(CONST_LibDir.'/Result.php');
+require_once(CONST_BasePath.'/lib/AddressDetails.php');
+require_once(CONST_BasePath.'/lib/Result.php');
 
 class PlaceLookup
 {
@@ -89,36 +89,20 @@ class PlaceLookup
     {
         $aParams = array();
 
-        if ($this->bAddressDetails) {
-            $aParams['addressdetails'] = '1';
-        }
-        if ($this->bExtraTags) {
-            $aParams['extratags'] = '1';
-        }
-        if ($this->bNameDetails) {
-            $aParams['namedetails'] = '1';
-        }
+        if ($this->bAddressDetails) $aParams['addressdetails'] = '1';
+        if ($this->bExtraTags) $aParams['extratags'] = '1';
+        if ($this->bNameDetails) $aParams['namedetails'] = '1';
 
-        if ($this->bIncludePolygonAsText) {
-            $aParams['polygon_text'] = '1';
-        }
-        if ($this->bIncludePolygonAsGeoJSON) {
-            $aParams['polygon_geojson'] = '1';
-        }
-        if ($this->bIncludePolygonAsKML) {
-            $aParams['polygon_kml'] = '1';
-        }
-        if ($this->bIncludePolygonAsSVG) {
-            $aParams['polygon_svg'] = '1';
-        }
+        if ($this->bIncludePolygonAsText) $aParams['polygon_text'] = '1';
+        if ($this->bIncludePolygonAsGeoJSON) $aParams['polygon_geojson'] = '1';
+        if ($this->bIncludePolygonAsKML) $aParams['polygon_kml'] = '1';
+        if ($this->bIncludePolygonAsSVG) $aParams['polygon_svg'] = '1';
 
         if ($this->fPolygonSimplificationThreshold > 0.0) {
             $aParams['polygon_threshold'] = $this->fPolygonSimplificationThreshold;
         }
 
-        if (!$this->bDeDupe) {
-            $aParams['dedupe'] = '0';
-        }
+        if (!$this->bDeDupe) $aParams['dedupe'] = '0';
 
         return $aParams;
     }
@@ -163,9 +147,8 @@ class PlaceLookup
 
     private function langAddressSql($sHousenumber)
     {
-        if ($this->bAddressDetails) {
+        if ($this->bAddressDetails)
             return ''; // langaddress will be computed from address details
-        }
 
         return 'get_address_by_language(place_id,'.$sHousenumber.','.$this->aLangPrefOrderSql.') AS langaddress,';
     }
@@ -251,20 +234,12 @@ class PlaceLookup
             $sSQL .= '     housenumber,';
             $sSQL .= '     country_code, ';
             $sSQL .= '     importance, ';
-            if (!$this->bDeDupe) {
-                $sSQL .= 'place_id,';
-            }
-            if (!$this->bAddressDetails) {
-                $sSQL .= 'langaddress, ';
-            }
+            if (!$this->bDeDupe) $sSQL .= 'place_id,';
+            if (!$this->bAddressDetails) $sSQL .= 'langaddress, ';
             $sSQL .= '     placename, ';
             $sSQL .= '     ref, ';
-            if ($this->bExtraTags) {
-                $sSQL .= 'extratags, ';
-            }
-            if ($this->bNameDetails) {
-                $sSQL .= 'name, ';
-            }
+            if ($this->bExtraTags) $sSQL .= 'extratags, ';
+            if ($this->bNameDetails) $sSQL .= 'name, ';
             $sSQL .= '     extra_place ';
 
             $aSubSelects[] = $sSQL;
@@ -285,12 +260,8 @@ class PlaceLookup
             $sSQL .= $this->langAddressSql('-1');
             $sSQL .= '  postcode as placename,';
             $sSQL .= '  postcode as ref,';
-            if ($this->bExtraTags) {
-                $sSQL .= 'null::text AS extra,';
-            }
-            if ($this->bNameDetails) {
-                $sSQL .= 'null::text AS names,';
-            }
+            if ($this->bExtraTags) $sSQL .= 'null::text AS extra,';
+            if ($this->bNameDetails) $sSQL .= 'null::text AS names,';
             $sSQL .= '  ST_x(geometry) AS lon, ST_y(geometry) AS lat,';
             $sSQL .= '  (0.75-(rank_search::float/40)) AS importance, ';
             $sSQL .= $this->addressImportanceSql('geometry', 'lp.parent_place_id');
@@ -327,12 +298,8 @@ class PlaceLookup
                     $sSQL .= $this->langAddressSql('housenumber_for_place');
                     $sSQL .= '     null::text AS placename, ';
                     $sSQL .= '     null::text AS ref, ';
-                    if ($this->bExtraTags) {
-                        $sSQL .= 'null::text AS extra,';
-                    }
-                    if ($this->bNameDetails) {
-                        $sSQL .= 'null::text AS names,';
-                    }
+                    if ($this->bExtraTags) $sSQL .= 'null::text AS extra,';
+                    if ($this->bNameDetails) $sSQL .= 'null::text AS names,';
                     $sSQL .= '     st_x(centroid) AS lon, ';
                     $sSQL .= '     st_y(centroid) AS lat,';
                     $sSQL .= '     -1.15 AS importance, ';
@@ -377,12 +344,8 @@ class PlaceLookup
                 $sSQL .= $this->langAddressSql('housenumber_for_place');
                 $sSQL .= '  null::text AS placename, ';
                 $sSQL .= '  null::text AS ref, ';
-                if ($this->bExtraTags) {
-                    $sSQL .= 'null::text AS extra, ';
-                }
-                if ($this->bNameDetails) {
-                    $sSQL .= 'null::text AS names, ';
-                }
+                if ($this->bExtraTags) $sSQL .= 'null::text AS extra, ';
+                if ($this->bNameDetails) $sSQL .= 'null::text AS names, ';
                 $sSQL .= '  st_x(centroid) AS lon, ';
                 $sSQL .= '  st_y(centroid) AS lat, ';
                 // slightly smaller than the importance for normal houses
@@ -410,6 +373,42 @@ class PlaceLookup
 
                 $aSubSelects[] = $sSQL;
             }
+
+            if (CONST_Use_Aux_Location_data) {
+                $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_AUX);
+                if ($sPlaceIDs) {
+                    $sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_AUX);
+                    $sSQL = '  SELECT ';
+                    $sSQL .= "     'L' AS osm_type, ";
+                    $sSQL .= '     place_id AS osm_id, ';
+                    $sSQL .= "     'place' AS class,";
+                    $sSQL .= "     'house' AS type, ";
+                    $sSQL .= '     null::smallint AS admin_level, ';
+                    $sSQL .= '     30 AS rank_search,';
+                    $sSQL .= '     30 AS rank_address, ';
+                    $sSQL .= '     place_id,';
+                    $sSQL .= '     parent_place_id, ';
+                    $sSQL .= '     housenumber,';
+                    $sSQL .= "     'us' AS country_code, ";
+                    $sSQL .= $this->langAddressSql('-1');
+                    $sSQL .= '     null::text AS placename, ';
+                    $sSQL .= '     null::text AS ref, ';
+                    if ($this->bExtraTags) $sSQL .= 'null::text AS extra, ';
+                    if ($this->bNameDetails) $sSQL .= 'null::text AS names, ';
+                    $sSQL .= '     ST_X(centroid) AS lon, ';
+                    $sSQL .= '     ST_Y(centroid) AS lat, ';
+                    $sSQL .= '     -1.10 AS importance, ';
+                    $sSQL .= $this->addressImportanceSql(
+                        'centroid',
+                        'location_property_aux.parent_place_id'
+                    );
+                    $sSQL .= '     null::text AS extra_place ';
+                    $sSQL .= '  FROM location_property_aux ';
+                    $sSQL .= "  WHERE place_id in ($sPlaceIDs) ";
+
+                    $aSubSelects[] = $sSQL;
+                }
+            }
         }
 
         if (empty($aSubSelects)) {
@@ -485,83 +484,67 @@ class PlaceLookup
     {
 
         $aOutlineResult = array();
-        if (!$iPlaceID) {
-            return $aOutlineResult;
-        }
+        if (!$iPlaceID) return $aOutlineResult;
 
-        // Get the bounding box and outline polygon
-        $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
-        if ($fLonReverse != null && $fLatReverse != null) {
-            $sSQL .= ' ST_Y(closest_point) as centrelat,';
-            $sSQL .= ' ST_X(closest_point) as centrelon,';
-        } else {
-            $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
-        }
-        $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
-        $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
-        if ($this->bIncludePolygonAsGeoJSON) {
-            $sSQL .= ',ST_AsGeoJSON(geometry) as asgeojson';
-        }
-        if ($this->bIncludePolygonAsKML) {
-            $sSQL .= ',ST_AsKML(geometry) as askml';
-        }
-        if ($this->bIncludePolygonAsSVG) {
-            $sSQL .= ',ST_AsSVG(geometry) as assvg';
-        }
-        if ($this->bIncludePolygonAsText) {
-            $sSQL .= ',ST_AsText(geometry) as astext';
-        }
-        if ($fLonReverse != null && $fLatReverse != null) {
-            $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
-            $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
-            $sFrom .=' ELSE centroid END AS closest_point';
-            $sFrom .= ' from placex where place_id = '.$iPlaceID.') as plx';
-        } else {
-            $sFrom = ' from placex where place_id = '.$iPlaceID;
-        }
-        if ($this->fPolygonSimplificationThreshold > 0) {
-            $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
-        } else {
-            $sSQL .= $sFrom;
-        }
-
-        $aPointPolygon = $this->oDB->getRow($sSQL, null, 'Could not get outline');
-
-        if ($aPointPolygon && $aPointPolygon['place_id']) {
-            if ($aPointPolygon['centrelon'] !== null && $aPointPolygon['centrelat'] !== null) {
-                $aOutlineResult['lat'] = $aPointPolygon['centrelat'];
-                $aOutlineResult['lon'] = $aPointPolygon['centrelon'];
+        if (CONST_Search_AreaPolygons) {
+            // Get the bounding box and outline polygon
+            $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
+            if ($fLonReverse != null && $fLatReverse != null) {
+                $sSQL .= ' ST_Y(closest_point) as centrelat,';
+                $sSQL .= ' ST_X(closest_point) as centrelon,';
+            } else {
+                $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
+            }
+            $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
+            $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
+            if ($this->bIncludePolygonAsGeoJSON) $sSQL .= ',ST_AsGeoJSON(geometry) as asgeojson';
+            if ($this->bIncludePolygonAsKML) $sSQL .= ',ST_AsKML(geometry) as askml';
+            if ($this->bIncludePolygonAsSVG) $sSQL .= ',ST_AsSVG(geometry) as assvg';
+            if ($this->bIncludePolygonAsText) $sSQL .= ',ST_AsText(geometry) as astext';
+            if ($fLonReverse != null && $fLatReverse != null) {
+                $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
+                $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
+                $sFrom .=' ELSE centroid END AS closest_point';
+                $sFrom .= ' from placex where place_id = '.$iPlaceID.') as plx';
+            } else {
+                $sFrom = ' from placex where place_id = '.$iPlaceID;
+            }
+            if ($this->fPolygonSimplificationThreshold > 0) {
+                $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
+            } else {
+                $sSQL .= $sFrom;
             }
 
-            if ($this->bIncludePolygonAsGeoJSON) {
-                $aOutlineResult['asgeojson'] = $aPointPolygon['asgeojson'];
-            }
-            if ($this->bIncludePolygonAsKML) {
-                $aOutlineResult['askml'] = $aPointPolygon['askml'];
-            }
-            if ($this->bIncludePolygonAsSVG) {
-                $aOutlineResult['assvg'] = $aPointPolygon['assvg'];
-            }
-            if ($this->bIncludePolygonAsText) {
-                $aOutlineResult['astext'] = $aPointPolygon['astext'];
-            }
+            $aPointPolygon = $this->oDB->getRow($sSQL, null, 'Could not get outline');
 
-            if (abs($aPointPolygon['minlat'] - $aPointPolygon['maxlat']) < 0.0000001) {
-                $aPointPolygon['minlat'] = $aPointPolygon['minlat'] - $fRadius;
-                $aPointPolygon['maxlat'] = $aPointPolygon['maxlat'] + $fRadius;
-            }
+            if ($aPointPolygon && $aPointPolygon['place_id']) {
+                if ($aPointPolygon['centrelon'] !== null && $aPointPolygon['centrelat'] !== null) {
+                    $aOutlineResult['lat'] = $aPointPolygon['centrelat'];
+                    $aOutlineResult['lon'] = $aPointPolygon['centrelon'];
+                }
 
-            if (abs($aPointPolygon['minlon'] - $aPointPolygon['maxlon']) < 0.0000001) {
-                $aPointPolygon['minlon'] = $aPointPolygon['minlon'] - $fRadius;
-                $aPointPolygon['maxlon'] = $aPointPolygon['maxlon'] + $fRadius;
-            }
+                if ($this->bIncludePolygonAsGeoJSON) $aOutlineResult['asgeojson'] = $aPointPolygon['asgeojson'];
+                if ($this->bIncludePolygonAsKML) $aOutlineResult['askml'] = $aPointPolygon['askml'];
+                if ($this->bIncludePolygonAsSVG) $aOutlineResult['assvg'] = $aPointPolygon['assvg'];
+                if ($this->bIncludePolygonAsText) $aOutlineResult['astext'] = $aPointPolygon['astext'];
 
-            $aOutlineResult['aBoundingBox'] = array(
-                                               (string)$aPointPolygon['minlat'],
-                                               (string)$aPointPolygon['maxlat'],
-                                               (string)$aPointPolygon['minlon'],
-                                               (string)$aPointPolygon['maxlon']
-                                              );
+                if (abs($aPointPolygon['minlat'] - $aPointPolygon['maxlat']) < 0.0000001) {
+                    $aPointPolygon['minlat'] = $aPointPolygon['minlat'] - $fRadius;
+                    $aPointPolygon['maxlat'] = $aPointPolygon['maxlat'] + $fRadius;
+                }
+
+                if (abs($aPointPolygon['minlon'] - $aPointPolygon['maxlon']) < 0.0000001) {
+                    $aPointPolygon['minlon'] = $aPointPolygon['minlon'] - $fRadius;
+                    $aPointPolygon['maxlon'] = $aPointPolygon['maxlon'] + $fRadius;
+                }
+
+                $aOutlineResult['aBoundingBox'] = array(
+                                                   (string)$aPointPolygon['minlat'],
+                                                   (string)$aPointPolygon['maxlat'],
+                                                   (string)$aPointPolygon['minlon'],
+                                                   (string)$aPointPolygon['maxlon']
+                                                  );
+            }
         }
 
         // as a fallback we generate a bounding box without knowing the size of the geometry

lib/Result.php

@@ -13,7 +13,8 @@ class Result
     const TABLE_PLACEX = 0;
     const TABLE_POSTCODE = 1;
     const TABLE_OSMLINE = 2;
-    const TABLE_TIGER = 3;
+    const TABLE_AUX = 3;
+    const TABLE_TIGER = 4;
 
     /// Database table that contains the result.
     public $iTable;
@@ -25,8 +26,6 @@ class Result
     public $iExactMatches = 0;
     /// Subranking within the results (the higher the worse).
     public $iResultRank = 0;
-    /// Address rank of the result.
-    public $iAddressRank;
 
     public function debugInfo()
     {
@@ -55,27 +54,6 @@ class Result
             }
         )));
     }
-
-    public static function joinIdsByTableMinRank($aResults, $iTable, $iMinAddressRank)
-    {
-        return join(',', array_keys(array_filter(
-            $aResults,
-            function ($aValue) use ($iTable, $iMinAddressRank) {
-                return $aValue->iTable == $iTable && $aValue->iAddressRank >= $iMinAddressRank;
-            }
-        )));
-    }
-
-    public static function joinIdsByTableMaxRank($aResults, $iTable, $iMaxAddressRank)
-    {
-        return join(',', array_keys(array_filter(
-            $aResults,
-            function ($aValue) use ($iTable, $iMaxAddressRank) {
-                return $aValue->iTable == $iTable && $aValue->iAddressRank <= $iMaxAddressRank;
-            }
-        )));
-    }
-
     public static function sqlHouseNumberTable($aResults, $iTable)
     {
         $sHousenumbers = '';
@@ -106,7 +84,7 @@ class Result
 
         foreach ($aResults as $oRes) {
             if ($oRes->iResultRank < $iMinRank) {
-                $aTail += $aHead;
+                $aTail = array_merge($aTail, $aHead);
                 $aHead = array($oRes->iId => $oRes);
                 $iMinRank = $oRes->iResultRank;
             } elseif ($oRes->iResultRank == $iMinRank) {

lib/ReverseGeocode.php

@@ -2,7 +2,7 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/Result.php');
+require_once(CONST_BasePath.'/lib/Result.php');
 
 class ReverseGeocode
 {
@@ -74,6 +74,8 @@ class ReverseGeocode
 
     protected function lookupLargeArea($sPointSQL, $iMaxRank)
     {
+        $oResult = null;
+
         if ($iMaxRank > 4) {
             $aPlace = $this->lookupPolygon($sPointSQL, $iMaxRank);
             if ($aPlace) {
@@ -165,13 +167,9 @@ class ReverseGeocode
     {
         Debug::newFunction('lookupPolygon');
         // polygon search begins at suburb-level
-        if ($iMaxRank > 25) {
-            $iMaxRank = 25;
-        }
+        if ($iMaxRank > 25) $iMaxRank = 25;
         // no polygon search over country-level
-        if ($iMaxRank < 5) {
-            $iMaxRank = 5;
-        }
+        if ($iMaxRank < 5) $iMaxRank = 5;
         // search for polygon
         $sSQL = 'SELECT place_id, parent_place_id, rank_address, rank_search FROM';
         $sSQL .= '(select place_id, parent_place_id, rank_address, rank_search, country_code, geometry';
@@ -192,6 +190,7 @@ class ReverseGeocode
 
         if ($aPoly) {
         // if a polygon is found, search for placenodes begins ...
+            $iParentPlaceID = $aPoly['parent_place_id'];
             $iRankAddress = $aPoly['rank_address'];
             $iRankSearch = $aPoly['rank_search'];
             $iPlaceID = $aPoly['place_id'];
@@ -243,24 +242,26 @@ class ReverseGeocode
     public function lookupPoint($sPointSQL, $bDoInterpolation = true)
     {
         Debug::newFunction('lookupPoint');
+        // starts if the search is on POI or street level,
+        // searches for the nearest POI or street,
+        // if a street is found and a POI is searched for,
+        // the nearest POI which the found street is a parent of is choosen.
+        $iMaxRank = $this->iMaxRank;
+
         // Find the nearest point
         $fSearchDiam = 0.006;
         $oResult = null;
         $aPlace = null;
 
         // for POI or street level
-        if ($this->iMaxRank >= 26) {
-            // starts if the search is on POI or street level,
-            // searches for the nearest POI or street,
-            // if a street is found and a POI is searched for,
-            // the nearest POI which the found street is a parent of is choosen.
+        if ($iMaxRank >= 26) {
             $sSQL = 'select place_id,parent_place_id,rank_address,country_code,';
             $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
             $sSQL .= ' FROM ';
             $sSQL .= ' placex';
             $sSQL .= '   WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
             $sSQL .= '   AND';
-            $sSQL .= ' rank_address between 26 and '.$this->iMaxRank;
+            $sSQL .= ' rank_address between 26 and '.$iMaxRank;
             $sSQL .= ' and (name is not null or housenumber is not null';
             $sSQL .= ' or rank_address between 26 and 27)';
             $sSQL .= ' and (rank_address between 26 and 27';
@@ -279,11 +280,34 @@ class ReverseGeocode
                 $iPlaceID = $aPlace['place_id'];
                 $oResult = new Result($iPlaceID);
                 $iRankAddress = $aPlace['rank_address'];
+                $iParentPlaceID = $aPlace['parent_place_id'];
+            }
+
+            if ($bDoInterpolation && $iMaxRank >= 30) {
+                $fDistance = $fSearchDiam;
+                if ($aPlace) {
+                    // We can't reliably go from the closest street to an
+                    // interpolation line because the closest interpolation
+                    // may have a different street segments as a parent.
+                    // Therefore allow an interpolation line to take precendence
+                    // even when the street is closer.
+                    $fDistance = $iRankAddress < 28 ? 0.001 : $aPlace['distance'];
+                }
+
+                $aHouse = $this->lookupInterpolation($sPointSQL, $fDistance);
+                Debug::printVar('Interpolation result', $aPlace);
+
+                if ($aHouse) {
+                    $oResult = new Result($aHouse['place_id'], Result::TABLE_OSMLINE);
+                    $oResult->iHouseNumber = closestHouseNumber($aHouse);
+                    $aPlace = $aHouse;
+                    $iRankAddress = 30;
+                }
             }
 
             if ($aPlace) {
                 // if street and maxrank > streetlevel
-                if ($iRankAddress <= 27 && $this->iMaxRank > 27) {
+                if ($iRankAddress <= 27 && $iMaxRank > 27) {
                     // find the closest object (up to a certain radius) of which the street is a parent of
                     $sSQL = ' select place_id,';
                     $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
@@ -304,9 +328,7 @@ class ReverseGeocode
                     Debug::printVar('Closest POI result', $aStreet);
 
                     if ($aStreet) {
-                        $aPlace = $aStreet;
                         $oResult = new Result($aStreet['place_id']);
-                        $iRankAddress = 30;
                     }
                 }
 
@@ -329,42 +351,17 @@ class ReverseGeocode
                     Debug::printVar('Tiger house number result', $aPlaceTiger);
 
                     if ($aPlaceTiger) {
-                        $aPlace = $aPlaceTiger;
                         $oResult = new Result($aPlaceTiger['place_id'], Result::TABLE_TIGER);
                         $oResult->iHouseNumber = closestHouseNumber($aPlaceTiger);
-                        $iRankAddress = 30;
                     }
                 }
-            }
-
-            if ($bDoInterpolation && $this->iMaxRank >= 30) {
-                $fDistance = $fSearchDiam;
-                if ($aPlace) {
-                    // We can't reliably go from the closest street to an
-                    // interpolation line because the closest interpolation
-                    // may have a different street segments as a parent.
-                    // Therefore allow an interpolation line to take precendence
-                    // even when the street is closer.
-                    $fDistance = $iRankAddress < 28 ? 0.001 : $aPlace['distance'];
-                }
-
-                $aHouse = $this->lookupInterpolation($sPointSQL, $fDistance);
-                Debug::printVar('Interpolation result', $aPlace);
-
-                if ($aHouse) {
-                    $oResult = new Result($aHouse['place_id'], Result::TABLE_OSMLINE);
-                    $oResult->iHouseNumber = closestHouseNumber($aHouse);
-                    $aPlace = $aHouse;
-                }
-            }
-
-            if (!$aPlace) {
+            } else {
                 // if no POI or street is found ...
                 $oResult = $this->lookupLargeArea($sPointSQL, 25);
             }
         } else {
             // lower than street level ($iMaxRank < 26 )
-            $oResult = $this->lookupLargeArea($sPointSQL, $this->iMaxRank);
+            $oResult = $this->lookupLargeArea($sPointSQL, $iMaxRank);
         }
 
         Debug::printVar('Final result', $oResult);

lib/SearchContext.php

@@ -2,7 +2,7 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/lib.php');
+require_once(CONST_BasePath.'/lib/lib.php');
 
 
 /**

lib/SearchDescription.php

@@ -2,9 +2,9 @@
 
 namespace Nominatim;
 
-require_once(CONST_LibDir.'/SpecialSearchOperator.php');
-require_once(CONST_LibDir.'/SearchContext.php');
-require_once(CONST_LibDir.'/Result.php');
+require_once(CONST_BasePath.'/lib/SpecialSearchOperator.php');
+require_once(CONST_BasePath.'/lib/SearchContext.php');
+require_once(CONST_BasePath.'/lib/Result.php');
 
 /**
  * Description of a single interpretation of a search query.
@@ -67,6 +67,47 @@ class SearchDescription
         return $this->iSearchRank;
     }
 
+    /**
+     * Make this search a POI search.
+     *
+     * In a POI search, objects are not (only) searched by their name
+     * but also by the primary OSM key/value pair (class and type in Nominatim).
+     *
+     * @param integer $iOperator Type of POI search
+     * @param string  $sClass    Class (or OSM tag key) of POI.
+     * @param string  $sType     Type (or OSM tag value) of POI.
+     *
+     * @return void
+     */
+    public function setPoiSearch($iOperator, $sClass, $sType)
+    {
+        $this->iOperator = $iOperator;
+        $this->sClass = $sClass;
+        $this->sType = $sType;
+    }
+
+    /**
+     * Check if this might be a full address search.
+     *
+     * @return bool True if the search contains name, address and housenumber.
+     */
+    public function looksLikeFullAddress()
+    {
+        return (!empty($this->aName))
+               && (!empty($this->aAddress) || $this->sCountryCode)
+               && preg_match('/[0-9]+/', $this->sHouseNumber);
+    }
+
+    /**
+     * Check if any operator is set.
+     *
+     * @return bool True, if this is a special search operation.
+     */
+    public function hasOperator()
+    {
+        return $this->iOperator != Operator::NONE;
+    }
+
     /**
      * Extract key/value pairs from a query.
      *
@@ -119,234 +160,252 @@ class SearchDescription
 
     /////////// Search building functions
 
-    /**
-     * Create a copy of this search description adding to search rank.
-     *
-     * @param integer $iTermCost  Cost to add to the current search rank.
-     *
-     * @return object Cloned search description.
-     */
-    public function clone($iTermCost)
-    {
-        $oSearch = clone $this;
-        $oSearch->iSearchRank += $iTermCost;
-
-        return $oSearch;
-    }
 
     /**
-     * Check if the search currently includes a name.
+     * Derive new searches by adding a full term to the existing search.
      *
-     * @param bool bIncludeNonNames  If true stop-word tokens are taken into
-     *                               account, too.
+     * @param object $oSearchTerm  Description of the token.
+     * @param bool   $bHasPartial  True if there are also tokens of partial terms
+     *                             with the same name.
+     * @param string $sPhraseType  Type of phrase the token is contained in.
+     * @param bool   $bFirstToken  True if the token is at the beginning of the
+     *                             query.
+     * @param bool   $bFirstPhrase True if the token is in the first phrase of
+     *                             the query.
+     * @param bool   $bLastToken   True if the token is at the end of the query.
      *
-     * @return bool True, if search has a name.
+     * @return SearchDescription[] List of derived search descriptions.
      */
-    public function hasName($bIncludeNonNames = false)
+    public function extendWithFullTerm($oSearchTerm, $bHasPartial, $sPhraseType, $bFirstToken, $bFirstPhrase, $bLastToken)
     {
-        return !empty($this->aName)
-               || (!empty($this->aNameNonSearch) && $bIncludeNonNames);
-    }
+        $aNewSearches = array();
 
-    /**
-     * Check if the search currently includes an address term.
-     *
-     * @return bool True, if any address term is included, including stop-word
-     *              terms.
-     */
-    public function hasAddress()
-    {
-        return !empty($this->aAddress) || !empty($this->aAddressNonSearch);
-    }
+        if (($sPhraseType == '' || $sPhraseType == 'country')
+            && is_a($oSearchTerm, '\Nominatim\Token\Country')
+        ) {
+            if (!$this->sCountryCode) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
+                $oSearch->sCountryCode = $oSearchTerm->sCountryCode;
+                // Country is almost always at the end of the string
+                // - increase score for finding it anywhere else (optimisation)
+                if (!$bLastToken) {
+                    $oSearch->iSearchRank += 5;
+                }
+                $aNewSearches[] = $oSearch;
+            }
+        } elseif (($sPhraseType == '' || $sPhraseType == 'postalcode')
+                  && is_a($oSearchTerm, '\Nominatim\Token\Postcode')
+        ) {
+            if (!$this->sPostcode) {
+                // If we have structured search or this is the first term,
+                // make the postcode the primary search element.
+                if ($this->iOperator == Operator::NONE && $bFirstToken) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank++;
+                    $oSearch->iOperator = Operator::POSTCODE;
+                    $oSearch->aAddress = array_merge($this->aAddress, $this->aName);
+                    $oSearch->aName =
+                        array($oSearchTerm->iId => $oSearchTerm->sPostcode);
+                    $aNewSearches[] = $oSearch;
+                }
 
-    /**
-     * Check if a country restriction is currently included in the search.
-     *
-     * @return bool True, if a country restriction is set.
-     */
-    public function hasCountry()
-    {
-        return $this->sCountryCode !== '';
-    }
+                // If we have a structured search or this is not the first term,
+                // add the postcode as an addendum.
+                if ($this->iOperator != Operator::POSTCODE
+                    && ($sPhraseType == 'postalcode' || !empty($this->aName))
+                ) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank++;
+                    if (strlen($oSearchTerm->sPostcode) < 4) {
+                        $oSearch->iSearchRank += 4 - strlen($oSearchTerm->sPostcode);
+                    }
+                    $oSearch->sPostcode = $oSearchTerm->sPostcode;
+                    $aNewSearches[] = $oSearch;
+                }
+            }
+        } elseif (($sPhraseType == '' || $sPhraseType == 'street')
+                 && is_a($oSearchTerm, '\Nominatim\Token\HouseNumber')
+        ) {
+            if (!$this->sHouseNumber && $this->iOperator != Operator::POSTCODE) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
+                $oSearch->sHouseNumber = $oSearchTerm->sToken;
+                // sanity check: if the housenumber is not mainly made
+                // up of numbers, add a penalty
+                if (preg_match('/\\d/', $oSearch->sHouseNumber) === 0
+                    || preg_match_all('/[^0-9]/', $oSearch->sHouseNumber, $aMatches) > 2) {
+                    $oSearch->iSearchRank++;
+                }
+                if (empty($oSearchTerm->iId)) {
+                    $oSearch->iSearchRank++;
+                }
+                // also must not appear in the middle of the address
+                if (!empty($this->aAddress)
+                    || (!empty($this->aAddressNonSearch))
+                    || $this->sPostcode
+                ) {
+                    $oSearch->iSearchRank++;
+                }
+                $aNewSearches[] = $oSearch;
+                // Housenumbers may appear in the name when the place has its own
+                // address terms.
+                if ($oSearchTerm->iId !== null
+                    && ($this->iNamePhrase >= 0 || empty($this->aName))
+                    && empty($this->aAddress)
+                   ) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank++;
+                    $oSearch->aAddress = $this->aName;
+                    $oSearch->bRareName = false;
+                    $oSearch->aName = array($oSearchTerm->iId => $oSearchTerm->iId);
+                    $aNewSearches[] = $oSearch;
+                }
+            }
+        } elseif ($sPhraseType == ''
+                  && is_a($oSearchTerm, '\Nominatim\Token\SpecialTerm')
+        ) {
+            if ($this->iOperator == Operator::NONE) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
 
-    /**
-     * Check if a postcode is currently included in the search.
-     *
-     * @return bool True, if a postcode is set.
-     */
-    public function hasPostcode()
-    {
-        return $this->sPostcode !== '';
-    }
+                $iOp = $oSearchTerm->iOperator;
+                if ($iOp == Operator::NONE) {
+                    if (!empty($this->aName) || $this->oContext->isBoundedSearch()) {
+                        $iOp = Operator::NAME;
+                    } else {
+                        $iOp = Operator::NEAR;
+                    }
+                    $oSearch->iSearchRank += 2;
+                }
 
-    /**
-     * Check if a house number is set for the search.
-     *
-     * @return bool True, if a house number is set.
-     */
-    public function hasHousenumber()
-    {
-        return $this->sHouseNumber !== '';
-    }
-
-    /**
-     * Check if a special type of place is requested.
-     *
-     * param integer iOperator  When set, check for the particular
-     *                          operator used for the special type.
-     *
-     * @return bool True, if speial type is requested or, if requested,
-     *              a special type with the given operator.
-     */
-    public function hasOperator($iOperator = null)
-    {
-        return $iOperator === null ? $this->iOperator != Operator::NONE : $this->iOperator == $iOperator;
-    }
-
-    /**
-     * Add the given token to the list of terms to search for in the address.
-     *
-     * @param integer iID       ID of term to add.
-     * @param bool bSearchable  Term should be used to search for result
-     *                          (i.e. term is not a stop word).
-     */
-    public function addAddressToken($iId, $bSearchable = true)
-    {
-        if ($bSearchable) {
-            $this->aAddress[$iId] = $iId;
-        } else {
-            $this->aAddressNonSearch[$iId] = $iId;
+                $oSearch->setPoiSearch(
+                    $iOp,
+                    $oSearchTerm->sClass,
+                    $oSearchTerm->sType
+                );
+                $aNewSearches[] = $oSearch;
+            }
+        } elseif ($sPhraseType != 'country'
+                  && is_a($oSearchTerm, '\Nominatim\Token\Word')
+        ) {
+            $iWordID = $oSearchTerm->iId;
+            // Full words can only be a name if they appear at the beginning
+            // of the phrase. In structured search the name must forcably in
+            // the first phrase. In unstructured search it may be in a later
+            // phrase when the first phrase is a house number.
+            if (!empty($this->aName) || !($bFirstPhrase || $sPhraseType == '')) {
+                if (($sPhraseType == '' || !$bFirstPhrase) && !$bHasPartial) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank += 3 * $oSearchTerm->iTermCount;
+                    $oSearch->aAddress[$iWordID] = $iWordID;
+                    $aNewSearches[] = $oSearch;
+                }
+            } else {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
+                $oSearch->aName = array($iWordID => $iWordID);
+                if (CONST_Search_NameOnlySearchFrequencyThreshold) {
+                    $oSearch->bRareName =
+                        $oSearchTerm->iSearchNameCount
+                          < CONST_Search_NameOnlySearchFrequencyThreshold;
+                }
+                $aNewSearches[] = $oSearch;
+            }
         }
+
+        return $aNewSearches;
     }
 
     /**
-     * Add the given full-word token to the list of terms to search for in the
-     * name.
+     * Derive new searches by adding a partial term to the existing search.
      *
-     * @param interger iId    ID of term to add.
-     * @param bool bRareName  True if the term is infrequent enough to not
-     *                        require other constraints for efficient search.
-     */
-    public function addNameToken($iId, $bRareName)
-    {
-        $this->aName[$iId] = $iId;
-        $this->bRareName = $bRareName;
-    }
-
-    /**
-     * Add the given partial token to the list of terms to search for in
-     * the name.
+     * @param string  $sToken             Term for the token.
+     * @param object  $oSearchTerm        Description of the token.
+     * @param bool    $bStructuredPhrases True if the search is structured.
+     * @param integer $iPhrase            Number of the phrase the token is in.
+     * @param array[] $aFullTokens        List of full term tokens with the
+     *                                    same name.
      *
-     * @param integer iID            ID of term to add.
-     * @param bool bSearchable       Term should be used to search for result
-     *                               (i.e. term is not a stop word).
-     * @param integer iPhraseNumber  Index of phrase, where the partial term
-     *                               appears.
+     * @return SearchDescription[] List of derived search descriptions.
      */
-    public function addPartialNameToken($iId, $bSearchable, $iPhraseNumber)
+    public function extendWithPartialTerm($sToken, $oSearchTerm, $bStructuredPhrases, $iPhrase, $aFullTokens)
     {
-        if ($bSearchable) {
-            $this->aName[$iId] = $iId;
-        } else {
-            $this->aNameNonSearch[$iId] = $iId;
+        // Only allow name terms.
+        if (!(is_a($oSearchTerm, '\Nominatim\Token\Word'))) {
+            return array();
         }
-        $this->iNamePhrase = $iPhraseNumber;
-    }
 
-    /**
-     * Set country restriction for the search.
-     *
-     * @param string sCountryCode  Country code of country to restrict search to.
-     */
-    public function setCountry($sCountryCode)
-    {
-        $this->sCountryCode = $sCountryCode;
-        $this->iNamePhrase = -1;
-    }
+        $aNewSearches = array();
+        $iWordID = $oSearchTerm->iId;
 
-    /**
-     * Set postcode search constraint.
-     *
-     * @param string sPostcode  Postcode the result should have.
-     */
-    public function setPostcode($sPostcode)
-    {
-        $this->sPostcode = $sPostcode;
-        $this->iNamePhrase = -1;
-    }
+        if ((!$bStructuredPhrases || $iPhrase > 0)
+            && (!empty($this->aName))
+            && strpos($sToken, ' ') === false
+        ) {
+            if ($oSearchTerm->iSearchNameCount < CONST_Max_Word_Frequency) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank += $oSearchTerm->iTermCount + 1;
+                if (empty($this->aName)) {
+                    $oSearch->iSearchRank++;
+                }
+                if (preg_match('#^[0-9]+$#', $sToken)) {
+                    $oSearch->iSearchRank++;
+                }
+                $oSearch->aAddress[$iWordID] = $iWordID;
+                $aNewSearches[] = $oSearch;
+            } else {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank += $oSearchTerm->iTermCount + 1;
+                $oSearch->aAddressNonSearch[$iWordID] = $iWordID;
+                if (!empty($aFullTokens)) {
+                    $oSearch->iSearchRank++;
+                }
+                $aNewSearches[] = $oSearch;
 
-    /**
-     * Make this search a search for a postcode object.
-     *
-     * @param integer iId       Token Id for the postcode.
-     * @param string sPostcode  Postcode to look for.
-     */
-    public function setPostcodeAsName($iId, $sPostcode)
-    {
-        $this->iOperator = Operator::POSTCODE;
-        $this->aAddress = array_merge($this->aAddress, $this->aName);
-        $this->aName = array($iId => $sPostcode);
-        $this->bRareName = true;
-        $this->iNamePhrase = -1;
-    }
+                // revert to the token version?
+                foreach ($aFullTokens as $oSearchTermToken) {
+                    if (is_a($oSearchTermToken, '\Nominatim\Token\Word')) {
+                        $oSearch = clone $this;
+                        $oSearch->iSearchRank += 3;
+                        $oSearch->aAddress[$oSearchTermToken->iId]
+                            = $oSearchTermToken->iId;
+                        $aNewSearches[] = $oSearch;
+                    }
+                }
+            }
+        }
 
-    /**
-     * Set house number search cnstraint.
-     *
-     * @param string sNumber  House number the result should have.
-     */
-    public function setHousenumber($sNumber)
-    {
-        $this->sHouseNumber = $sNumber;
-        $this->iNamePhrase = -1;
-    }
+        if ((!$this->sPostcode && !$this->aAddress && !$this->aAddressNonSearch)
+            && (empty($this->aName) || $this->iNamePhrase == $iPhrase)
+        ) {
+            $oSearch = clone $this;
+            $oSearch->iSearchRank += 2;
+            if (empty($this->aName)) {
+                $oSearch->iSearchRank += 1;
+            }
+            if (preg_match('#^[0-9]+$#', $sToken)) {
+                $oSearch->iSearchRank += 2;
+            }
+            if ($oSearchTerm->iSearchNameCount < CONST_Max_Word_Frequency) {
+                if (empty($this->aName)
+                    && CONST_Search_NameOnlySearchFrequencyThreshold
+                ) {
+                    $oSearch->bRareName =
+                        $oSearchTerm->iSearchNameCount
+                          < CONST_Search_NameOnlySearchFrequencyThreshold;
+                } else {
+                    $oSearch->bRareName = false;
+                }
+                $oSearch->aName[$iWordID] = $iWordID;
+            } else {
+                $oSearch->aNameNonSearch[$iWordID] = $iWordID;
+            }
+            $oSearch->iNamePhrase = $iPhrase;
+            $aNewSearches[] = $oSearch;
+        }
 
-    /**
-     * Make this search a search for a house number.
-     *
-     * @param integer iId  Token Id for the house number.
-     */
-    public function setHousenumberAsName($iId)
-    {
-        $this->aAddress = array_merge($this->aAddress, $this->aName);
-        $this->bRareName = false;
-        $this->aName = array($iId => $iId);
-        $this->iNamePhrase = -1;
-    }
-
-    /**
-     * Make this search a POI search.
-     *
-     * In a POI search, objects are not (only) searched by their name
-     * but also by the primary OSM key/value pair (class and type in Nominatim).
-     *
-     * @param integer $iOperator Type of POI search
-     * @param string  $sClass    Class (or OSM tag key) of POI.
-     * @param string  $sType     Type (or OSM tag value) of POI.
-     *
-     * @return void
-     */
-    public function setPoiSearch($iOperator, $sClass, $sType)
-    {
-        $this->iOperator = $iOperator;
-        $this->sClass = $sClass;
-        $this->sType = $sType;
-        $this->iNamePhrase = -1;
-    }
-
-    public function getNamePhrase()
-    {
-        return $this->iNamePhrase;
-    }
-
-    /**
-     * Get the global search context.
-     *
-     * @return object  Objects of global search constraints.
-     */
-    public function getContext()
-    {
-        return $this->oContext;
+        return $aNewSearches;
     }
 
     /////////// Query functions
@@ -367,6 +426,7 @@ class SearchDescription
     public function query(&$oDB, $iMinRank, $iMaxRank, $iLimit)
     {
         $aResults = array();
+        $iHousenumber = -1;
 
         if ($this->sCountryCode
             && empty($this->aName)
@@ -399,24 +459,19 @@ class SearchDescription
 
             // Now search for housenumber, if housenumber provided. Can be zero.
             if (($this->sHouseNumber || $this->sHouseNumber === '0') && !empty($aResults)) {
-                $aHnResults = $this->queryHouseNumber($oDB, $aResults);
-
                 // Downgrade the rank of the street results, they are missing
-                // the housenumber. Also drop POI places (rank 30) here, they
-                // cannot be a parent place and therefore must not be shown
-                // as a result for a search with a missing housenumber.
+                // the housenumber.
                 foreach ($aResults as $oRes) {
-                    if ($oRes->iAddressRank < 28) {
-                        if ($oRes->iAddressRank >= 26) {
-                            $oRes->iResultRank++;
-                        } else {
-                            $oRes->iResultRank += 2;
-                        }
-                        $aHnResults[$oRes->iId] = $oRes;
-                    }
+                    $oRes->iResultRank++;
                 }
 
-                $aResults = $aHnResults;
+                $aHnResults = $this->queryHouseNumber($oDB, $aResults);
+
+                if (!empty($aHnResults)) {
+                    foreach ($aHnResults as $oRes) {
+                        $aResults[$oRes->iId] = $oRes;
+                    }
+                }
             }
 
             // finally get POIs if requested
@@ -568,14 +623,14 @@ class SearchDescription
         // too many results are expected for the street, i.e. if the result
         // will be narrowed down by an address. Remeber that with ordering
         // every single result has to be checked.
-        if ($this->sHouseNumber && ($this->bRareName || !empty($this->aAddress) || $this->sPostcode)) {
+        if ($this->sHouseNumber && (!empty($this->aAddress) || $this->sPostcode)) {
             $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
             $aOrder[] = ' (';
             $aOrder[0] .= 'EXISTS(';
             $aOrder[0] .= '  SELECT place_id';
             $aOrder[0] .= '  FROM placex';
             $aOrder[0] .= '  WHERE parent_place_id = search_name.place_id';
-            $aOrder[0] .= "    AND housenumber ~* E'".$sHouseNumberRegex."'";
+            $aOrder[0] .= "    AND transliteration(housenumber) ~* E'".$sHouseNumberRegex."'";
             $aOrder[0] .= '  LIMIT 1';
             $aOrder[0] .= ') ';
             // also housenumbers from interpolation lines table are needed
@@ -672,7 +727,7 @@ class SearchDescription
         $aResults = array();
 
         if (!empty($aTerms)) {
-            $sSQL = 'SELECT place_id, address_rank,'.$sExactMatchSQL;
+            $sSQL = 'SELECT place_id,'.$sExactMatchSQL;
             $sSQL .= ' FROM search_name';
             $sSQL .= ' WHERE '.join(' and ', $aTerms);
             $sSQL .= ' ORDER BY '.join(', ', $aOrder);
@@ -685,7 +740,6 @@ class SearchDescription
             foreach ($aDBResults as $aResult) {
                 $oResult = new Result($aResult['place_id']);
                 $oResult->iExactMatches = $aResult['exactmatch'];
-                $oResult->iAddressRank = $aResult['address_rank'];
                 $aResults[$aResult['place_id']] = $oResult;
             }
         }
@@ -696,33 +750,16 @@ class SearchDescription
     private function queryHouseNumber(&$oDB, $aRoadPlaceIDs)
     {
         $aResults = array();
-        $sRoadPlaceIDs = Result::joinIdsByTableMaxRank(
-            $aRoadPlaceIDs,
-            Result::TABLE_PLACEX,
-            27
-        );
-        $sPOIPlaceIDs = Result::joinIdsByTableMinRank(
-            $aRoadPlaceIDs,
-            Result::TABLE_PLACEX,
-            30
-        );
+        $sPlaceIDs = Result::joinIdsByTable($aRoadPlaceIDs, Result::TABLE_PLACEX);
 
-        $aIDCondition = array();
-        if ($sRoadPlaceIDs) {
-            $aIDCondition[] = 'parent_place_id in ('.$sRoadPlaceIDs.')';
-        }
-        if ($sPOIPlaceIDs) {
-            $aIDCondition[] = 'place_id in ('.$sPOIPlaceIDs.')';
-        }
-
-        if (empty($aIDCondition)) {
+        if (!$sPlaceIDs) {
             return $aResults;
         }
 
         $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
-        $sSQL = 'SELECT place_id FROM placex WHERE';
-        $sSQL .= "  housenumber ~* E'".$sHouseNumberRegex."'";
-        $sSQL .= ' AND ('.join(' OR ', $aIDCondition).')';
+        $sSQL = 'SELECT place_id FROM placex ';
+        $sSQL .= 'WHERE parent_place_id in ('.$sPlaceIDs.')';
+        $sSQL .= "  AND transliteration(housenumber) ~* E'".$sHouseNumberRegex."'";
         $sSQL .= $this->oContext->excludeSQL(' AND place_id');
 
         Debug::printSQL($sSQL);
@@ -734,11 +771,11 @@ class SearchDescription
 
         $bIsIntHouseNumber= (bool) preg_match('/[0-9]+/', $this->sHouseNumber);
         $iHousenumber = intval($this->sHouseNumber);
-        if ($bIsIntHouseNumber && $sRoadPlaceIDs && empty($aResults)) {
+        if ($bIsIntHouseNumber && empty($aResults)) {
             // if nothing found, search in the interpolation line table
             $sSQL = 'SELECT distinct place_id FROM location_property_osmline';
             $sSQL .= ' WHERE startnumber is not NULL';
-            $sSQL .= '  AND parent_place_id in ('.$sRoadPlaceIDs.') AND (';
+            $sSQL .= '  AND parent_place_id in ('.$sPlaceIDs.') AND (';
             if ($iHousenumber % 2 == 0) {
                 // If housenumber is even, look for housenumber in streets
                 // with interpolationtype even or all.
@@ -761,10 +798,24 @@ class SearchDescription
             }
         }
 
+        // If nothing found try the aux fallback table
+        if (CONST_Use_Aux_Location_data && empty($aResults)) {
+            $sSQL = 'SELECT place_id FROM location_property_aux';
+            $sSQL .= ' WHERE parent_place_id in ('.$sPlaceIDs.')';
+            $sSQL .= " AND housenumber = '".$this->sHouseNumber."'";
+            $sSQL .= $this->oContext->excludeSQL(' AND place_id');
+
+            Debug::printSQL($sSQL);
+
+            foreach ($oDB->getCol($sSQL) as $iPlaceId) {
+                $aResults[$iPlaceId] = new Result($iPlaceId, Result::TABLE_AUX);
+            }
+        }
+
         // If nothing found then search in Tiger data (location_property_tiger)
-        if (CONST_Use_US_Tiger_Data && $sRoadPlaceIDs && $bIsIntHouseNumber && empty($aResults)) {
+        if (CONST_Use_US_Tiger_Data && $bIsIntHouseNumber && empty($aResults)) {
             $sSQL = 'SELECT place_id FROM location_property_tiger';
-            $sSQL .= ' WHERE parent_place_id in ('.$sRoadPlaceIDs.') and (';
+            $sSQL .= ' WHERE parent_place_id in ('.$sPlaceIDs.') and (';
             if ($iHousenumber % 2 == 0) {
                 $sSQL .= "interpolationtype='even'";
             } else {
@@ -976,7 +1027,7 @@ class SearchDescription
                 'Name terms (stop words)' => $this->aNameNonSearch,
                 'Address terms' => $this->aAddress,
                 'Address terms (stop words)' => $this->aAddressNonSearch,
-                'Address terms (full words)' => $this->aFullNameAddress ?? '',
+                'Address terms (full words)' => $this->aFullNameAddress,
                 'Special search' => $this->iOperator,
                 'Class' => $this->sClass,
                 'Type' => $this->sType,
@@ -988,7 +1039,7 @@ class SearchDescription
     public function dumpAsHtmlTableRow(&$aWordIDs)
     {
         $kf = function ($k) use (&$aWordIDs) {
-            return $aWordIDs[$k] ?? '['.$k.']';
+            return $aWordIDs[$k];
         };
 
         echo '<tr>';

lib/Shell.php

@@ -7,7 +7,7 @@ class Shell
     public function __construct($sBaseCmd, ...$aParams)
     {
         if (!$sBaseCmd) {
-            throw new \Exception('Command missing in new() call');
+            throw new Exception('Command missing in new() call');
         }
         $this->baseCmd = $sBaseCmd;
         $this->aParams = array();
@@ -33,9 +33,7 @@ class Shell
     public function addEnvPair($sKey, $sVal)
     {
         if (isset($sKey) && $sKey && isset($sVal)) {
-            if (!isset($this->aEnv)) {
-                $this->aEnv = $_ENV;
-            }
+            if (!isset($this->aEnv)) $this->aEnv = $_ENV;
             $this->aEnv = array_merge($this->aEnv, array($sKey => $sVal), $_ENV);
         }
         return $this;
@@ -50,7 +48,7 @@ class Shell
         return join(' ', $aEscaped);
     }
 
-    public function run($bExitOnFail = false)
+    public function run()
     {
         $sCmd = $this->escapedCmd();
         // $aEnv does not need escaping, proc_open seems to handle it fine
@@ -69,16 +67,14 @@ class Shell
         fclose($aPipes[0]); // no stdin
 
         $iStat = proc_close($hProc);
-
-        if ($iStat != 0 && $bExitOnFail) {
-            exit($iStat);
-        }
-
         return $iStat;
     }
 
+
+
     private function escapeParam($sParam)
     {
-        return (preg_match('/^-*\w+$/', $sParam)) ? $sParam : escapeshellarg($sParam);
+        if (preg_match('/^-*\w+$/', $sParam)) return $sParam;
+        return escapeshellarg($sParam);
     }
 }

lib/Status.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace Nominatim;
+
+use Exception;
+
+class Status
+{
+    protected $oDB;
+
+    public function __construct(&$oDB)
+    {
+        $this->oDB =& $oDB;
+    }
+
+    public function status()
+    {
+        if (!$this->oDB) {
+            throw new Exception('No database', 700);
+        }
+
+        try {
+            $this->oDB->connect();
+        } catch (\Nominatim\DatabaseError $e) {
+            throw new Exception('Database connection failed', 700);
+        }
+
+        $sStandardWord = $this->oDB->getOne("SELECT make_standard_name('a')");
+        if ($sStandardWord === false) {
+            throw new Exception('Module failed', 701);
+        }
+
+        if ($sStandardWord != 'a') {
+            throw new Exception('Module call failed', 702);
+        }
+
+        $sSQL = 'SELECT word_id, word_token, word, class, type, country_code, ';
+        $sSQL .= "operator, search_name_count FROM word WHERE word_token IN (' a')";
+        $iWordID = $this->oDB->getOne($sSQL);
+        if ($iWordID === false) {
+            throw new Exception('Query failed', 703);
+        }
+        if (!$iWordID) {
+            throw new Exception('No value', 704);
+        }
+    }
+
+    public function dataDate()
+    {
+        $sSQL = 'SELECT EXTRACT(EPOCH FROM lastimportdate) FROM import_status LIMIT 1';
+        $iDataDateEpoch = $this->oDB->getOne($sSQL);
+
+        if ($iDataDateEpoch === false) {
+            throw Exception('Data date query failed '.$iDataDateEpoch->getMessage(), 705);
+        }
+
+        return $iDataDateEpoch;
+    }
+}

lib/TokenCountry.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Nominatim\Token;
+
+/**
+ * A country token.
+ */
+class Country
+{
+    /// Database word id, if available.
+    public $iId;
+    /// Two-letter country code (lower-cased).
+    public $sCountryCode;
+
+    public function __construct($iId, $sCountryCode)
+    {
+        $this->iId = $iId;
+        $this->sCountryCode = $sCountryCode;
+    }
+
+    public function debugInfo()
+    {
+        return array(
+                'ID' => $this->iId,
+                'Type' => 'country',
+                'Info' => $this->sCountryCode
+               );
+    }
+}

lib/TokenHousenumber.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Nominatim\Token;
+
+/**
+ * A house number token.
+ */
+class HouseNumber
+{
+    /// Database word id, if available.
+    public $iId;
+    /// Normalized house number.
+    public $sToken;
+
+    public function __construct($iId, $sToken)
+    {
+        $this->iId = $iId;
+        $this->sToken = $sToken;
+    }
+
+    public function debugInfo()
+    {
+        return array(
+                'ID' => $this->iId,
+                'Type' => 'house number',
+                'Info' => array('nr' => $this->sToken)
+               );
+    }
+}

lib/TokenList.php

@@ -0,0 +1,216 @@
+<?php
+
+namespace Nominatim;
+
+require_once(CONST_BasePath.'/lib/TokenCountry.php');
+require_once(CONST_BasePath.'/lib/TokenHousenumber.php');
+require_once(CONST_BasePath.'/lib/TokenPostcode.php');
+require_once(CONST_BasePath.'/lib/TokenSpecialTerm.php');
+require_once(CONST_BasePath.'/lib/TokenWord.php');
+require_once(CONST_BasePath.'/lib/SpecialSearchOperator.php');
+
+/**
+ * Saves information about the tokens that appear in a search query.
+ *
+ * Tokens are sorted by their normalized form, the token word. There are different
+ * kinds of tokens, represented by different Token* classes. Note that
+ * tokens do not have a common base class. All tokens need to have a field
+ * with the word id that points to an entry in the `word` database table
+ * but otherwise the information saved about a token can be very different.
+ *
+ * There are two different kinds of token words: full words and partial terms.
+ *
+ * Full words start with a space. They represent a complete name of a place.
+ * All special tokens are normally full words.
+ *
+ * Partial terms have no space at the beginning. They may represent a part of
+ * a name of a place (e.g. in the name 'World Trade Center' a partial term
+ * would be 'Trade' or 'Trade Center'). They are only used in TokenWord.
+ */
+class TokenList
+{
+    // List of list of tokens indexed by their word_token.
+    private $aTokens = array();
+
+
+    /**
+     * Return total number of tokens.
+     *
+     * @return Integer
+     */
+    public function count()
+    {
+        return count($this->aTokens);
+    }
+
+    /**
+     * Check if there are tokens for the given token word.
+     *
+     * @param string $sWord Token word to look for.
+     *
+     * @return bool True if there is one or more token for the token word.
+     */
+    public function contains($sWord)
+    {
+        return isset($this->aTokens[$sWord]);
+    }
+
+    /**
+     * Check if there are partial or full tokens for the given word.
+     *
+     * @param string $sWord Token word to look for.
+     *
+     * @return bool True if there is one or more token for the token word.
+     */
+    public function containsAny($sWord)
+    {
+        return isset($this->aTokens[$sWord]) || isset($this->aTokens[' '.$sWord]);
+    }
+
+    /**
+     * Get the list of tokens for the given token word.
+     *
+     * @param string $sWord Token word to look for.
+     *
+     * @return object[] Array of tokens for the given token word or an
+     *                  empty array if no tokens could be found.
+     */
+    public function get($sWord)
+    {
+        return isset($this->aTokens[$sWord]) ? $this->aTokens[$sWord] : array();
+    }
+
+    public function getFullWordIDs()
+    {
+        $ids = array();
+
+        foreach ($this->aTokens as $aTokenList) {
+            foreach ($aTokenList as $oToken) {
+                if (is_a($oToken, '\Nominatim\Token\Word') && !$oToken->bPartial) {
+                    $ids[$oToken->iId] = $oToken->iId;
+                }
+            }
+        }
+
+        return $ids;
+    }
+
+    /**
+     * Add token information from the word table in the database.
+     *
+     * @param object   $oDB           Nominatim::DB instance.
+     * @param string[] $aTokens       List of tokens to look up in the database.
+     * @param string[] $aCountryCodes List of country restrictions.
+     * @param string   $sNormQuery    Normalized query string.
+     * @param object   $oNormalizer   Normalizer function to use on tokens.
+     *
+     * @return void
+     */
+    public function addTokensFromDB(&$oDB, &$aTokens, &$aCountryCodes, $sNormQuery, $oNormalizer)
+    {
+        // Check which tokens we have, get the ID numbers
+        $sSQL = 'SELECT word_id, word_token, word, class, type, country_code,';
+        $sSQL .= ' operator, coalesce(search_name_count, 0) as count';
+        $sSQL .= ' FROM word WHERE word_token in (';
+        $sSQL .= join(',', $oDB->getDBQuotedList($aTokens)).')';
+
+        Debug::printSQL($sSQL);
+
+        $aDBWords = $oDB->getAll($sSQL, null, 'Could not get word tokens.');
+
+        foreach ($aDBWords as $aWord) {
+            $oToken = null;
+            $iId = (int) $aWord['word_id'];
+
+            if ($aWord['class']) {
+                // Special terms need to appear in their normalized form.
+                if ($aWord['word']) {
+                    $sNormWord = $aWord['word'];
+                    if ($oNormalizer != null) {
+                        $sNormWord = $oNormalizer->transliterate($aWord['word']);
+                    }
+                    if (strpos($sNormQuery, $sNormWord) === false) {
+                        continue;
+                    }
+                }
+
+                if ($aWord['class'] == 'place' && $aWord['type'] == 'house') {
+                    $oToken = new Token\HouseNumber($iId, trim($aWord['word_token']));
+                } elseif ($aWord['class'] == 'place' && $aWord['type'] == 'postcode') {
+                    if ($aWord['word']
+                        && pg_escape_string($aWord['word']) == $aWord['word']
+                    ) {
+                        $oToken = new Token\Postcode(
+                            $iId,
+                            $aWord['word'],
+                            $aWord['country_code']
+                        );
+                    }
+                } else {
+                    // near and in operator the same at the moment
+                    $oToken = new Token\SpecialTerm(
+                        $iId,
+                        $aWord['class'],
+                        $aWord['type'],
+                        $aWord['operator'] ? Operator::NEAR : Operator::NONE
+                    );
+                }
+            } elseif ($aWord['country_code']) {
+                // Filter country tokens that do not match restricted countries.
+                if (!$aCountryCodes
+                    || in_array($aWord['country_code'], $aCountryCodes)
+                ) {
+                    $oToken = new Token\Country($iId, $aWord['country_code']);
+                }
+            } else {
+                $oToken = new Token\Word(
+                    $iId,
+                    $aWord['word_token'][0] != ' ',
+                    (int) $aWord['count'],
+                    substr_count($aWord['word_token'], ' ')
+                );
+            }
+
+            if ($oToken) {
+                $this->addToken($aWord['word_token'], $oToken);
+            }
+        }
+    }
+
+    /**
+     * Add a new token for the given word.
+     *
+     * @param string $sWord  Word the token describes.
+     * @param object $oToken Token object to add.
+     *
+     * @return void
+     */
+    public function addToken($sWord, $oToken)
+    {
+        if (isset($this->aTokens[$sWord])) {
+            $this->aTokens[$sWord][] = $oToken;
+        } else {
+            $this->aTokens[$sWord] = array($oToken);
+        }
+    }
+
+    public function debugTokenByWordIdList()
+    {
+        $aWordsIDs = array();
+        foreach ($this->aTokens as $sToken => $aWords) {
+            foreach ($aWords as $aToken) {
+                if ($aToken->iId !== null) {
+                    $aWordsIDs[$aToken->iId] =
+                        '#'.$sToken.'('.$aToken->iId.')#';
+                }
+            }
+        }
+
+        return $aWordsIDs;
+    }
+
+    public function debugInfo()
+    {
+        return $this->aTokens;
+    }
+}

lib/TokenPostcode.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace Nominatim\Token;
+
+/**
+ * A postcode token.
+ */
+class Postcode
+{
+    /// Database word id, if available.
+    public $iId;
+    /// Full nomralized postcode (upper cased).
+    public $sPostcode;
+    // Optional country code the postcode belongs to (currently unused).
+    public $sCountryCode;
+
+    public function __construct($iId, $sPostcode, $sCountryCode = '')
+    {
+        $this->iId = $iId;
+        $this->sPostcode = $sPostcode;
+        $this->sCountryCode = empty($sCountryCode) ? '' : $sCountryCode;
+    }
+
+    public function debugInfo()
+    {
+        return array(
+                'ID' => $this->iId,
+                'Type' => 'postcode',
+                'Info' => $this->sPostcode.'('.$this->sCountryCode.')'
+               );
+    }
+}

lib/TokenSpecialTerm.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace Nominatim\Token;
+
+require_once(CONST_BasePath.'/lib/SpecialSearchOperator.php');
+
+/**
+ * A word token describing a place type.
+ */
+class SpecialTerm
+{
+    /// Database word id, if applicable.
+    public $iId;
+    /// Class (or OSM tag key) of the place to look for.
+    public $sClass;
+    /// Type (or OSM tag value) of the place to look for.
+    public $sType;
+    /// Relationship of the operator to the object (see Operator class).
+    public $iOperator;
+
+    public function __construct($iID, $sClass, $sType, $iOperator)
+    {
+        $this->iId = $iID;
+        $this->sClass = $sClass;
+        $this->sType = $sType;
+        $this->iOperator = $iOperator;
+    }
+
+    public function debugInfo()
+    {
+        return array(
+                'ID' => $this->iId,
+                'Type' => 'special term',
+                'Info' => array(
+                           'class' => $this->sClass,
+                           'type' => $this->sType,
+                           'operator' => \Nominatim\Operator::toString($this->iOperator)
+                          )
+               );
+    }
+}

lib/TokenWord.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace Nominatim\Token;
+
+/**
+ * A standard word token.
+ */
+class Word
+{
+    /// Database word id, if applicable.
+    public $iId;
+    /// If true, the word may represent only part of a place name.
+    public $bPartial;
+    /// Number of appearances in the database.
+    public $iSearchNameCount;
+    /// Number of terms in the word.
+    public $iTermCount;
+
+    public function __construct($iId, $bPartial, $iSearchNameCount, $iTermCount)
+    {
+        $this->iId = $iId;
+        $this->bPartial = $bPartial;
+        $this->iSearchNameCount = $iSearchNameCount;
+        $this->iTermCount = $iTermCount;
+    }
+
+    public function debugInfo()
+    {
+        return array(
+                'ID' => $this->iId,
+                'Type' => 'word',
+                'Info' => array(
+                           'partial' => $this->bPartial,
+                           'count' => $this->iSearchNameCount
+                          )
+               );
+    }
+}

lib/cmd.php

@@ -1,6 +1,6 @@
 <?php
 
-require_once(CONST_LibDir.'/Shell.php');
+require_once(CONST_BasePath.'/lib/Shell.php');
 
 function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnknown = false)
 {
@@ -9,12 +9,8 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
 
     foreach ($aSpec as $aLine) {
         if (is_array($aLine)) {
-            if ($aLine[0]) {
-                $aQuick['--'.$aLine[0]] = $aLine;
-            }
-            if ($aLine[1]) {
-                $aQuick['-'.$aLine[1]] = $aLine;
-            }
+            if ($aLine[0]) $aQuick['--'.$aLine[0]] = $aLine;
+            if ($aLine[1]) $aQuick['-'.$aLine[1]] = $aLine;
             $aCounts[$aLine[0]] = 0;
         }
     }
@@ -32,9 +28,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
                     $xVal = array();
                     for ($n = $aLine[4]; $i < $iSize && $n; $n--) {
                         $i++;
-                        if ($i >= $iSize || $aArg[$i][0] == '-') {
-                            showUsage($aSpec, $bExitOnError, 'Parameter of  \''.$aLine[0].'\' is missing');
-                        }
+                        if ($i >= $iSize || $aArg[$i][0] == '-') showUsage($aSpec, $bExitOnError, 'Parameter of  \''.$aLine[0].'\' is missing');
 
                         switch ($aLine[6]) {
                             case 'realpath':
@@ -62,9 +56,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
                                 break;
                         }
                     }
-                    if ($aLine[4] == 1) {
-                        $xVal = $xVal[0];
-                    }
+                    if ($aLine[4] == 1) $xVal = $xVal[0];
                 } else {
                     $xVal = true;
                 }
@@ -73,9 +65,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
             }
 
             if ($aLine[3] > 1) {
-                if (!array_key_exists($aLine[0], $aResult)) {
-                    $aResult[$aLine[0]] = array();
-                }
+                if (!array_key_exists($aLine[0], $aResult)) $aResult[$aLine[0]] = array();
                 $aResult[$aLine[0]][] = $xVal;
             } else {
                 $aResult[$aLine[0]] = $xVal;
@@ -85,23 +75,18 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
         }
     }
 
-    if (array_key_exists('help', $aResult)) {
-        showUsage($aSpec);
-    }
-    if ($bUnknown && $bExitOnUnknown) {
-        showUsage($aSpec, $bExitOnError, 'Unknown option \''.$bUnknown.'\'');
-    }
+    if (array_key_exists('help', $aResult)) showUsage($aSpec);
+    if ($bUnknown && $bExitOnUnknown) showUsage($aSpec, $bExitOnError, 'Unknown option \''.$bUnknown.'\'');
 
     foreach ($aSpec as $aLine) {
         if (is_array($aLine)) {
-            if ($aCounts[$aLine[0]] < $aLine[2]) {
-                showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is missing');
-            }
-            if ($aCounts[$aLine[0]] > $aLine[3]) {
-                showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is pressent too many times');
-            }
-            if ($aLine[6] == 'bool' && !array_key_exists($aLine[0], $aResult)) {
-                $aResult[$aLine[0]] = false;
+            if ($aCounts[$aLine[0]] < $aLine[2]) showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is missing');
+            if ($aCounts[$aLine[0]] > $aLine[3]) showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is pressent too many times');
+            switch ($aLine[6]) {
+                case 'bool':
+                    if (!array_key_exists($aLine[0], $aResult))
+                        $aResult[$aLine[0]] = false;
+                    break;
             }
         }
     }
@@ -124,12 +109,8 @@ function showUsage($aSpec, $bExit = false, $sError = false)
                 echo "\n";
             }
             $aNames = array();
-            if ($aLine[1]) {
-                $aNames[] = '-'.$aLine[1];
-            }
-            if ($aLine[0]) {
-                $aNames[] = '--'.$aLine[0];
-            }
+            if ($aLine[1]) $aNames[] = '-'.$aLine[1];
+            if ($aLine[0]) $aNames[] = '--'.$aLine[0];
             $sName = join(', ', $aNames);
             echo '  '.$sName.str_repeat(' ', 30-strlen($sName)).$aLine[7]."\n";
         } else {
@@ -163,29 +144,54 @@ function repeatWarnings()
 }
 
 
-function setupHTTPProxy()
+function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreErrors = false)
 {
-    if (!getSettingBool('HTTP_PROXY')) {
-        return;
+    // Convert database DSN to psql parameters
+    $aDSNInfo = \Nominatim\DB::parseDSN(CONST_Database_DSN);
+    if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+
+    $oCmd = new \Nominatim\Shell('psql');
+    $oCmd->addParams('--port', $aDSNInfo['port']);
+    $oCmd->addParams('--dbname', $aDSNInfo['database']);
+    if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+        $oCmd->addParams('--host', $aDSNInfo['hostspec']);
+    }
+    if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+        $oCmd->addParams('--username', $aDSNInfo['username']);
+    }
+    if (isset($aDSNInfo['password'])) {
+        $oCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
+    }
+    if (!$bVerbose) {
+        $oCmd->addParams('--quiet');
+    }
+    if ($bfatal && !$bIgnoreErrors) {
+        $oCmd->addParams('-v', 'ON_ERROR_STOP=1');
     }
 
-    $sProxy = 'tcp://'.getSetting('HTTP_PROXY_HOST').':'.getSetting('HTTP_PROXY_PROT');
-    $aHeaders = array();
-
-    $sLogin = getSetting('HTTP_PROXY_LOGIN');
-    $sPassword = getSetting('HTTP_PROXY_PASSWORD');
-
-    if ($sLogin && $sPassword) {
-        $sAuth = base64_encode($sLogin.':'.$sPassword);
-        $aHeaders = array('Proxy-Authorization: Basic '.$sAuth);
-    }
-
-    $aProxyHeader = array(
-                     'proxy' => $sProxy,
-                     'request_fulluri' => true,
-                     'header' => $aHeaders
+    $aDescriptors = array(
+                     0 => array('pipe', 'r'),
+                     1 => STDOUT,
+                     2 => STDERR
                     );
+    $ahPipes = null;
+    $hProcess = @proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
+    if (!is_resource($hProcess)) {
+        fail('unable to start pgsql');
+    }
 
-    $aContext = array('http' => $aProxyHeader, 'https' => $aProxyHeader);
-    stream_context_set_default($aContext);
+    if (!$bVerbose) {
+        fwrite($ahPipes[0], 'set client_min_messages to WARNING;');
+    }
+
+    while (strlen($sScript)) {
+        $iWritten = fwrite($ahPipes[0], $sScript);
+        if ($iWritten <= 0) break;
+        $sScript = substr($sScript, $iWritten);
+    }
+    fclose($ahPipes[0]);
+    $iReturn = proc_close($hProcess);
+    if ($bfatal && $iReturn > 0) {
+        fail("pgsql returned with error code ($iReturn)");
+    }
 }

lib/init-cmd.php

@@ -0,0 +1,28 @@
+<?php
+
+require_once('init.php');
+require_once('cmd.php');
+require_once('DebugNone.php');
+
+// handle http proxy when using file_get_contents
+if (CONST_HTTP_Proxy) {
+    $proxy = 'tcp://' . CONST_HTTP_Proxy_Host . ':' . CONST_HTTP_Proxy_Port;
+    $aHeaders = array();
+    if (CONST_HTTP_Proxy_Login != null && CONST_HTTP_Proxy_Login != '' && CONST_HTTP_Proxy_Password != null && CONST_HTTP_Proxy_Password != '') {
+        $auth = base64_encode(CONST_HTTP_Proxy_Login . ':' . CONST_HTTP_Proxy_Password);
+        $aHeaders = array("Proxy-Authorization: Basic $auth");
+    }
+    $aContext = array(
+                 'http' => array(
+                            'proxy' => $proxy,
+                            'request_fulluri' => true,
+                            'header' => $aHeaders
+                           ),
+                 'https' => array(
+                             'proxy' => $proxy,
+                             'request_fulluri' => true,
+                             'header' => $aHeaders
+                            )
+                );
+    stream_context_set_default($aContext);
+}

lib/init-website.php

@@ -12,7 +12,7 @@ require_once(CONST_Debug ? 'DebugHtml.php' : 'DebugNone.php');
 
 function userError($sMsg)
 {
-    throw new \Exception($sMsg, 400);
+    throw new Exception($sMsg, 400);
 }
 
 
@@ -20,7 +20,7 @@ function exception_handler_json($exception)
 {
     http_response_code($exception->getCode());
     header('Content-type: application/json; charset=utf-8');
-    include(CONST_LibDir.'/template/error-json.php');
+    include(CONST_BasePath.'/lib/template/error-json.php');
     exit();
 }
 
@@ -29,7 +29,7 @@ function exception_handler_xml($exception)
     http_response_code($exception->getCode());
     header('Content-type: text/xml; charset=utf-8');
     echo '<?xml version="1.0" encoding="UTF-8" ?>'."\n";
-    include(CONST_LibDir.'/template/error-xml.php');
+    include(CONST_BasePath.'/lib/template/error-xml.php');
     exit();
 }
 
@@ -37,7 +37,7 @@ function shutdown_exception_handler_xml()
 {
     $error = error_get_last();
     if ($error !== null && $error['type'] === E_ERROR) {
-        exception_handler_xml(new \Exception($error['message'], 500));
+        exception_handler_xml(new Exception($error['message'], 500));
     }
 }
 
@@ -45,7 +45,7 @@ function shutdown_exception_handler_json()
 {
     $error = error_get_last();
     if ($error !== null && $error['type'] === E_ERROR) {
-        exception_handler_json(new \Exception($error['message'], 500));
+        exception_handler_json(new Exception($error['message'], 500));
     }
 }
 
@@ -81,10 +81,6 @@ if (CONST_NoAccessControl) {
         header('Access-Control-Allow-Headers: '.$_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS']);
     }
 }
-if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] == 'OPTIONS') {
-    exit;
-}
+if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
 
-if (CONST_Debug) {
-    header('Content-type: text/html; charset=utf-8');
-}
+if (CONST_Debug) header('Content-type: text/html; charset=utf-8');

lib/init.php

@@ -0,0 +1,4 @@
+<?php
+
+require_once(CONST_BasePath.'/lib/lib.php');
+require_once(CONST_BasePath.'/lib/DB.php');

lib/lib.php

@@ -1,42 +1,10 @@
 <?php
 
-function loadSettings($sProjectDir)
-{
-    @define('CONST_InstallDir', $sProjectDir);
-    // Temporary hack to set the direcory via environment instead of
-    // the installed scripts. Neither setting is part of the official
-    // set of settings.
-    defined('CONST_ConfigDir') or define('CONST_ConfigDir', $_SERVER['NOMINATIM_CONFIGDIR']);
-}
-
-function getSetting($sConfName, $sDefault = null)
-{
-    $sValue = $_SERVER['NOMINATIM_'.$sConfName];
-
-    if ($sDefault !== null && !$sValue) {
-        return $sDefault;
-    }
-
-    return $sValue;
-}
-
-function getSettingBool($sConfName)
-{
-    $sVal = strtolower(getSetting($sConfName));
-
-    return strcmp($sVal, 'yes') == 0
-           || strcmp($sVal, 'true') == 0
-           || strcmp($sVal, '1') == 0;
-}
-
 function fail($sError, $sUserError = false)
 {
-    if (!$sUserError) {
-        $sUserError = $sError;
-    }
+    if (!$sUserError) $sUserError = $sError;
     error_log('ERROR: '.$sError);
-    var_dump($sUserError);
-    echo "\n";
+    var_dump($sUserError)."\n";
     exit(-1);
 }
 
@@ -84,9 +52,8 @@ function getDatabaseDate(&$oDB)
 
 function byImportance($a, $b)
 {
-    if ($a['importance'] != $b['importance']) {
+    if ($a['importance'] != $b['importance'])
         return ($a['importance'] > $b['importance']?-1:1);
-    }
 
     return $a['foundorder'] <=> $b['foundorder'];
 }
@@ -198,6 +165,17 @@ function parseLatLon($sQuery)
     return array($sFound, $fQueryLat, $fQueryLon);
 }
 
+function createPointsAroundCenter($fLon, $fLat, $fRadius)
+{
+    $iSteps = max(8, min(100, ($fRadius * 40000)^2));
+    $fStepSize = (2*pi())/$iSteps;
+    $aPolyPoints = array();
+    for ($f = 0; $f < 2*pi(); $f += $fStepSize) {
+        $aPolyPoints[] = array('', $fLon+($fRadius*sin($f)), $fLat+($fRadius*cos($f)) );
+    }
+    return $aPolyPoints;
+}
+
 function closestHouseNumber($aRow)
 {
     $fHouse = $aRow['startnumber']
@@ -218,11 +196,24 @@ function closestHouseNumber($aRow)
     return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
 }
 
-if (!function_exists('array_key_last')) {
-    function array_key_last(array $array)
-    {
-        if (!empty($array)) {
-            return key(array_slice($array, -1, 1, true));
-        }
-    }
+function getSearchRankLabel($iRank)
+{
+    if (!isset($iRank)) return 'unknown';
+    if ($iRank < 2) return 'continent';
+    if ($iRank < 4) return 'sea';
+    if ($iRank < 8) return 'country';
+    if ($iRank < 12) return 'state';
+    if ($iRank < 16) return 'county';
+    if ($iRank == 16) return 'city';
+    if ($iRank == 17) return 'town / island';
+    if ($iRank == 18) return 'village / hamlet';
+    if ($iRank == 20) return 'suburb';
+    if ($iRank == 21) return 'postcode area';
+    if ($iRank == 22) return 'croft / farm / locality / islet';
+    if ($iRank == 23) return 'postcode area';
+    if ($iRank == 25) return 'postcode point';
+    if ($iRank == 26) return 'street / major landmark';
+    if ($iRank == 27) return 'minory street / path';
+    if ($iRank == 28) return 'house / building';
+    return 'other: ' . $iRank;
 }

lib/log.php

@@ -5,23 +5,15 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
 {
     $fStartTime = microtime(true);
     $aStartTime = explode('.', $fStartTime);
-    if (!isset($aStartTime[1])) {
-        $aStartTime[1] = '0';
-    }
+    if (!isset($aStartTime[1])) $aStartTime[1] = '0';
 
     $sOutputFormat = '';
-    if (isset($_GET['format'])) {
-        $sOutputFormat = $_GET['format'];
-    }
+    if (isset($_GET['format'])) $sOutputFormat = $_GET['format'];
 
     if ($sType == 'reverse') {
         $sOutQuery = (isset($_GET['lat'])?$_GET['lat']:'').'/';
-        if (isset($_GET['lon'])) {
-            $sOutQuery .= $_GET['lon'];
-        }
-        if (isset($_GET['zoom'])) {
-            $sOutQuery .= '/'.$_GET['zoom'];
-        }
+        if (isset($_GET['lon'])) $sOutQuery .= $_GET['lon'];
+        if (isset($_GET['zoom'])) $sOutQuery .= '/'.$_GET['zoom'];
     } else {
         $sOutQuery = $sQuery;
     }
@@ -36,15 +28,13 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
             );
 
     if (CONST_Log_DB) {
-        if (isset($_GET['email'])) {
+        if (isset($_GET['email']))
             $sUserAgent = $_GET['email'];
-        } elseif (isset($_SERVER['HTTP_REFERER'])) {
+        elseif (isset($_SERVER['HTTP_REFERER']))
             $sUserAgent = $_SERVER['HTTP_REFERER'];
-        } elseif (isset($_SERVER['HTTP_USER_AGENT'])) {
+        elseif (isset($_SERVER['HTTP_USER_AGENT']))
             $sUserAgent = $_SERVER['HTTP_USER_AGENT'];
-        } else {
-            $sUserAgent = '';
-        }
+        else $sUserAgent = '';
         $sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format,searchterm)';
         $sSQL .= ' values (';
         $sSQL .= join(',', $oDB->getDBQuotedList(array(
@@ -70,9 +60,7 @@ function logEnd(&$oDB, $hLog, $iNumResults)
 
     if (CONST_Log_DB) {
         $aEndTime = explode('.', $fEndTime);
-        if (!$aEndTime[1]) {
-            $aEndTime[1] = '0';
-        }
+        if (!$aEndTime[1]) $aEndTime[1] = '0';
         $sEndTime = date('Y-m-d H:i:s', $aEndTime[0]).'.'.$aEndTime[1];
 
         $sSQL = 'update new_query_log set endtime = '.$oDB->getDBQuoted($sEndTime).', results = '.$iNumResults;

lib/output.php

@@ -0,0 +1,73 @@
+<?php
+
+
+function formatOSMType($sType, $bIncludeExternal = true)
+{
+    if ($sType == 'N') return 'node';
+    if ($sType == 'W') return 'way';
+    if ($sType == 'R') return 'relation';
+
+    if (!$bIncludeExternal) return '';
+
+    if ($sType == 'T') return 'way';
+    if ($sType == 'I') return 'way';
+
+    // not handled: P, L
+
+    return '';
+}
+
+function osmLink($aFeature, $sRefText = false)
+{
+    $sOSMType = formatOSMType($aFeature['osm_type'], false);
+    if ($sOSMType) {
+        return '<a href="//www.openstreetmap.org/'.$sOSMType.'/'.$aFeature['osm_id'].'">'.$sOSMType.' '.($sRefText?$sRefText:$aFeature['osm_id']).'</a>';
+    }
+    return '';
+}
+
+function wikipediaLink($aFeature)
+{
+    if ($aFeature['wikipedia']) {
+        list($sLanguage, $sArticle) = explode(':', $aFeature['wikipedia']);
+        return '<a href="https://'.$sLanguage.'.wikipedia.org/wiki/'.urlencode($sArticle).'" target="_blank">'.$aFeature['wikipedia'].'</a>';
+    }
+    return '';
+}
+
+function detailsLink($aFeature, $sTitle = false, $sExtraProperties = false)
+{
+    if (!$aFeature['place_id']) return '';
+
+    $sHtml = '<a ';
+    if ($sExtraProperties) {
+        $sHtml .= $sExtraProperties.' ';
+    }
+
+    $sHtml .= 'href="details.php?place_id='.$aFeature['place_id'].'">'.($sTitle?$sTitle:$aFeature['place_id']).'</a>';
+
+    return $sHtml;
+}
+
+function detailsPermaLink($aFeature, $sRefText = false, $sExtraProperties = false)
+{
+    $sOSMType = formatOSMType($aFeature['osm_type'], false);
+
+    if ($sOSMType) {
+        $sHtml = '<a ';
+        if ($sExtraProperties) {
+            $sHtml .= $sExtraProperties.' ';
+        }
+        $sHtml .= 'href="details.php?osmtype='.$aFeature['osm_type']
+                  .'&osmid='.$aFeature['osm_id'].'&class='.$aFeature['class'].'">';
+
+        if ($sRefText) {
+            $sHtml .= $sRefText.'</a>';
+        } else {
+            $sHtml .= $sOSMType.' '.$aFeature['osm_id'].'</a>';
+        }
+
+        return $sHtml;
+    }
+    return detailsLink($aFeature, $sRefText, $sExtraProperties);
+}

lib/setup/AddressLevelParser.php

@@ -0,0 +1,98 @@
+<?php
+
+namespace Nominatim\Setup;
+
+/**
+ * Parses an address level description.
+ */
+class AddressLevelParser
+{
+    private $aLevels;
+
+    public function __construct($sDescriptionFile)
+    {
+        $sJson = file_get_contents($sDescriptionFile);
+        $this->aLevels = json_decode($sJson, true);
+        if (!$this->aLevels) {
+            switch (json_last_error()) {
+                case JSON_ERROR_NONE:
+                    break;
+                case JSON_ERROR_DEPTH:
+                    fail('JSON error - Maximum stack depth exceeded');
+                    break;
+                case JSON_ERROR_STATE_MISMATCH:
+                    fail('JSON error - Underflow or the modes mismatch');
+                    break;
+                case JSON_ERROR_CTRL_CHAR:
+                    fail('JSON error - Unexpected control character found');
+                    break;
+                case JSON_ERROR_SYNTAX:
+                    fail('JSON error - Syntax error, malformed JSON');
+                    break;
+                case JSON_ERROR_UTF8:
+                    fail('JSON error - Malformed UTF-8 characters, possibly incorrectly encoded');
+                    break;
+                default:
+                    fail('JSON error - Unknown error');
+                    break;
+            }
+        }
+    }
+
+    /**
+     * Dump the description into a database table.
+     *
+     * @param object $oDB    Database conneciton to use.
+     * @param string $sTable Name of table to create.
+     *
+     * @return null
+     *
+     * A new table is created. Any previously existing table is dropped.
+     * The table has the following columns:
+     * country, class, type, rank_search, rank_address.
+     */
+    public function createTable($oDB, $sTable)
+    {
+        $oDB->exec('DROP TABLE IF EXISTS '.$sTable);
+        $sSql = 'CREATE TABLE '.$sTable;
+        $sSql .= '(country_code varchar(2), class TEXT, type TEXT,';
+        $sSql .= ' rank_search SMALLINT, rank_address SMALLINT)';
+        $oDB->exec($sSql);
+
+        $sSql = 'CREATE UNIQUE INDEX ON '.$sTable.' (country_code, class, type)';
+        $oDB->exec($sSql);
+
+        $sSql = 'INSERT INTO '.$sTable.' VALUES ';
+        foreach ($this->aLevels as $aLevel) {
+            $aCountries = array();
+            if (isset($aLevel['countries'])) {
+                foreach ($aLevel['countries'] as $sCountry) {
+                    $aCountries[$sCountry] = $oDB->getDBQuoted($sCountry);
+                }
+            } else {
+                $aCountries['NULL'] = 'NULL';
+            }
+            foreach ($aLevel['tags'] as $sKey => $aValues) {
+                foreach ($aValues as $sValue => $mRanks) {
+                    $aFields = array(
+                        $oDB->getDBQuoted($sKey),
+                        $sValue ? $oDB->getDBQuoted($sValue) : 'NULL'
+                    );
+                    if (is_array($mRanks)) {
+                        $aFields[] = (string) $mRanks[0];
+                        $aFields[] = (string) $mRanks[1];
+                    } else {
+                        $aFields[] = (string) $mRanks;
+                        $aFields[] = (string) $mRanks;
+                    }
+                    $sLine = ','.join(',', $aFields).'),';
+
+                    foreach ($aCountries as $sCountries) {
+                        $sSql .= '('.$sCountries.$sLine;
+                    }
+                }
+            }
+        }
+        $oDB->exec(rtrim($sSql, ','));
+    }
+}