hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-07 02:24:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3592 from lonvia/remove-cmake

Browse Source 
Remove cmake building and bundled osm2pgsql
This commit is contained in:
Sarah Hoffmann
2024-11-19 10:02:53 +01:00
committed by GitHub
parent 02364ce6c8 4e59efa178
commit c172ca8c6c
16 changed files with 141 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
.github/actions/build-nominatim/action.yml vendored
View File

@@ -1,18 +1,10 @@
name: 'Build Nominatim' name: 'Build Nominatim'
inputs: inputs:
flavour: dependencies:
description: 'Version of Ubuntu to install on' description: 'Where to install dependencies from (pip/apt)'
required: false required: false
default: 'ubuntu-20' default: 'pip'
cmake-args:
description: 'Additional options to hand to cmake'
required: false
default: ''
lua:
description: 'Version of Lua to use'
required: false
default: '5.3'
runs: runs:
using: "composite" using: "composite"
@@ -23,30 +15,30 @@ runs:
sudo rm -rf /opt/hostedtoolcache/go /opt/hostedtoolcache/CodeQL /usr/lib/jvm /usr/local/share/chromium /usr/local/lib/android sudo rm -rf /opt/hostedtoolcache/go /opt/hostedtoolcache/CodeQL /usr/lib/jvm /usr/local/share/chromium /usr/local/lib/android
df -h df -h
shell: bash shell: bash
- name: Install${{ matrix.flavour }} prerequisites - name: Install general prerequisites
run: | run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson nlohmann-json3-dev libspatialite-dev libsqlite3-mod-spatialite sudo apt-get install -y -qq libspatialite-dev libsqlite3-mod-spatialite libicu-dev virtualenv python3-dev osm2pgsql
if [ "$FLAVOUR" == "oldstuff" ]; then
pip3 install MarkupSafe==2.0.1 python-dotenv jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4.31 psycopg==3.1.7 datrie asyncpg aiosqlite
else
sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-dotenv python3-yaml
pip3 install sqlalchemy psycopg aiosqlite
fi
shell: bash shell: bash
env:
FLAVOUR: ${{ inputs.flavour }}
CMAKE_ARGS: ${{ inputs.cmake-args }}
LUA_VERSION: ${{ inputs.lua }}
- name: Configure - name: Install prerequisites from apt
run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim
shell: bash
env:
CMAKE_ARGS: ${{ inputs.cmake-args }}
- name: Build
run: | run: |
make -j2 all sudo apt-get install -y -qq python3-icu python3-datrie python3-jinja2 python3-psutil python3-dotenv python3-yaml python3-sqlalchemy python3-psycopg python3-asyncpg
sudo make install shell: bash
if: inputs.dependencies == 'apt'
- name: Setup virtual environment (for pip)
run: |
virtualenv venv
./venv/bin/pip install -U pip
shell: bash
if: inputs.dependencies == 'pip'
- name: Setup virtual environment (for apt)
run: |
virtualenv venv --system-site-packages
shell: bash
if: inputs.dependencies == 'apt'
- name: Build nominatim
run: ./venv/bin/pip install Nominatim/packaging/nominatim-{api,db}
shell: bash shell: bash
working-directory: build

6
.github/actions/setup-postgresql/action.yml vendored
View File

@@ -4,9 +4,6 @@ inputs:
postgresql-version: postgresql-version:
description: 'Version of PostgreSQL to install' description: 'Version of PostgreSQL to install'
required: true required: true
postgis-version:
description: 'Version of Postgis to install'
required: true
runs: runs:
using: "composite" using: "composite"
@@ -24,11 +21,10 @@ runs:
- name: Install PostgreSQL - name: Install PostgreSQL
run: | run: |
sudo apt-get install -y -qq --no-install-suggests --no-install-recommends postgresql-client-${PGVER} postgresql-${PGVER}-postgis-${POSTGISVER} postgresql-${PGVER}-postgis-${POSTGISVER}-scripts postgresql-contrib-${PGVER} postgresql-${PGVER} sudo apt-get install -y -qq --no-install-suggests --no-install-recommends postgresql-client-${PGVER} postgresql-${PGVER}-postgis-3 postgresql-${PGVER}-postgis-3-scripts postgresql-contrib-${PGVER} postgresql-${PGVER}
shell: bash shell: bash
env: env:
PGVER: ${{ inputs.postgresql-version }} PGVER: ${{ inputs.postgresql-version }}
POSTGISVER: ${{ inputs.postgis-version }}
- name: Adapt postgresql configuration - name: Adapt postgresql configuration
run: | run: |

98
.github/workflows/ci-tests.yml vendored
View File

@@ -42,13 +42,13 @@ jobs:
- flavour: ubuntu-20 - flavour: ubuntu-20
ubuntu: 20 ubuntu: 20
postgresql: 12 postgresql: 12
postgis: 3
lua: '5.1' lua: '5.1'
dependencies: pip
- flavour: ubuntu-24 - flavour: ubuntu-24
ubuntu: 24 ubuntu: 24
postgresql: 17 postgresql: 17
postgis: 3
lua: '5.3' lua: '5.3'
dependencies: apt
runs-on: ubuntu-${{ matrix.ubuntu }}.04 runs-on: ubuntu-${{ matrix.ubuntu }}.04
@@ -60,58 +60,67 @@ jobs:
- name: Unpack Nominatim - name: Unpack Nominatim
run: tar xf nominatim-src.tar.bz2 run: tar xf nominatim-src.tar.bz2
- uses: actions/setup-python@v5
with:
python-version: 3.7
if: matrix.flavour == 'oldstuff'
- uses: ./Nominatim/.github/actions/setup-postgresql - uses: ./Nominatim/.github/actions/setup-postgresql
with: with:
postgresql-version: ${{ matrix.postgresql }} postgresql-version: ${{ matrix.postgresql }}
postgis-version: ${{ matrix.postgis }}
- uses: ./Nominatim/.github/actions/build-nominatim - uses: ./Nominatim/.github/actions/build-nominatim
with: with:
flavour: ${{ matrix.flavour }} dependencies: ${{ matrix.dependencies }}
lua: ${{ matrix.lua }}
- name: Compile osm2pgsql
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev
mkdir osm2pgsql-build
cd osm2pgsql-build
git clone https://github.com/osm2pgsql-dev/osm2pgsql
mkdir build
cd build
cmake ../osm2pgsql
make
sudo make install
cd ../..
rm -rf osm2pgsql-build
if: matrix.ubuntu == '20'
env:
LUA_VERSION: ${{ matrix.lua }}
- name: Install test prerequisites - name: Install test prerequisites
run: pip3 install behave==1.2.6 run: ./venv/bin/pip install behave==1.2.6
- name: Install test prerequisites - name: Install test prerequisites (apt)
run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn python3-falcon python3-aiosqlite python3-pyosmium
if: matrix.dependencies == 'apt'
- name: Install newer pytest-asyncio - name: Install test prerequisites (pip)
run: pip3 install -U pytest-asyncio run: ./venv/bin/pip install pytest-asyncio falcon starlette asgi_lifespan aiosqlite osmium uvicorn
if: matrix.flavour == 'ubuntu-20' if: matrix.dependencies == 'pip'
- name: Install Python webservers
run: pip3 install falcon starlette asgi_lifespan
- name: Install latest flake8 - name: Install latest flake8
run: pip3 install -U flake8 run: ./venv/bin/pip install -U flake8
- name: Python linting - name: Python linting
run: python3 -m flake8 src run: ../venv/bin/python -m flake8 src
working-directory: Nominatim working-directory: Nominatim
- name: Install mypy and typechecking info
run: ./venv/bin/pip install -U mypy types-PyYAML types-jinja2 types-psutil types-requests types-ujson types-Pygments typing-extensions
if: matrix.dependencies == 'pip'
- name: Python static typechecking
run: ../venv/bin/python -m mypy --strict --python-version 3.8 src
working-directory: Nominatim
if: matrix.dependencies == 'pip'
- name: Python unit tests - name: Python unit tests
run: python3 -m pytest test/python run: ../venv/bin/python -m pytest test/python
working-directory: Nominatim working-directory: Nominatim
- name: BDD tests - name: BDD tests
run: | run: |
export PATH=$GITHUB_WORKSPACE/build/osm2pgsql:$PATH ../../../venv/bin/python -m behave -DREMOVE_TEMPLATE=1 --format=progress3
python3 -m behave -DREMOVE_TEMPLATE=1 --format=progress3
working-directory: Nominatim/test/bdd working-directory: Nominatim/test/bdd
- name: Install mypy and typechecking info
run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson types-Pygments typing-extensions
- name: Python static typechecking
run: python3 -m mypy --strict --python-version 3.8 src
working-directory: Nominatim
install: install:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: create-archive needs: create-archive
@@ -186,12 +195,10 @@ jobs:
- name: Add nominatim to path - name: Add nominatim to path
run: | run: |
sudo ln -s /home/nominatim/nominatim-venv/bin/nominatim /usr/local/bin/nominatim sudo ln -s /home/nominatim/nominatim-venv/bin/nominatim /usr/local/bin/nominatim
if: matrix.ubuntu == 24
- name: Need lua binary - name: Need lua binary
run: | run: |
sudo apt-get install -y lua5.4 lua-dkjson sudo apt-get install -y lua5.4 lua-dkjson
if: matrix.ubuntu == 24
- name: Print version - name: Print version
run: nominatim --version run: nominatim --version
@@ -221,19 +228,9 @@ jobs:
run: nominatim admin --warm run: nominatim admin --warm
working-directory: /home/nominatim/nominatim-project working-directory: /home/nominatim/nominatim-project
- name: Prepare update (Ubuntu) - name: Install osmium
run: apt-get install -y python3-pip
shell: bash
- name: Install osmium (Ubuntu 22)
run: |
pip3 install --user osmium
if: matrix.ubuntu == 22
- name: Install osmium (Ubuntu 24)
run: | run: |
/home/nominatim/nominatim-venv/bin/pip install osmium /home/nominatim/nominatim-venv/bin/pip install osmium
if: matrix.ubuntu == 24
- name: Run update - name: Run update
run: | run: |
@@ -260,7 +257,7 @@ jobs:
working-directory: /home/nominatim/nominatim-project working-directory: /home/nominatim/nominatim-project
install-no-superuser: install-no-superuser:
runs-on: ubuntu-latest runs-on: ubuntu-24.04
needs: create-archive needs: create-archive
steps: steps:
@@ -274,12 +271,8 @@ jobs:
- uses: ./Nominatim/.github/actions/setup-postgresql - uses: ./Nominatim/.github/actions/setup-postgresql
with: with:
postgresql-version: 16 postgresql-version: 16
postgis-version: 3
- uses: ./Nominatim/.github/actions/build-nominatim - uses: ./Nominatim/.github/actions/build-nominatim
with:
flavour: ubuntu-22
lua: 5.3
- name: Prepare import environment - name: Prepare import environment
run: | run: |
@@ -288,7 +281,7 @@ jobs:
- name: Prepare Database - name: Prepare Database
run: | run: |
nominatim import --prepare-database ./venv/bin/nominatim import --prepare-database
- name: Create import user - name: Create import user
run: | run: |
@@ -298,10 +291,10 @@ jobs:
- name: Run import - name: Run import
run: | run: |
NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" nominatim import --continue import-from-file --osm-file test.pbf NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" ./venv/bin/nominatim import --continue import-from-file --osm-file test.pbf
- name: Check full import - name: Check full import
run: nominatim admin --check-database run: ./venv/bin/nominatim admin --check-database
migrate: migrate:
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
@@ -318,7 +311,6 @@ jobs:
- uses: ./Nominatim/.github/actions/setup-postgresql - uses: ./Nominatim/.github/actions/setup-postgresql
with: with:
postgresql-version: 17 postgresql-version: 17
postgis-version: 3
- name: Install Python dependencies - name: Install Python dependencies
run: | run: |

4
.gitmodules vendored
View File

@@ -1,4 +0,0 @@
[submodule "osm2pgsql"]
path = osm2pgsql
url = https://github.com/openstreetmap/osm2pgsql.git
ignore = dirty

216
CMakeLists.txt
View File

@@ -1,216 +0,0 @@
#-----------------------------------------------------------------------------
#
# CMake Config
#
# Nominatim
#
#-----------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
#-----------------------------------------------------------------------------
#
# Project version
#
#-----------------------------------------------------------------------------
project(nominatim)
set(NOMINATIM_VERSION_MAJOR 4)
set(NOMINATIM_VERSION_MINOR 5)
set(NOMINATIM_VERSION_PATCH 0)
set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
add_definitions(-DNOMINATIM_VERSION="${NOMINATIM_VERSION}")
# Setting GIT_HASH
find_package(Git)
if (GIT_FOUND)
execute_process(
COMMAND "${GIT_EXECUTABLE}" log -1 --format=%h
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
OUTPUT_VARIABLE GIT_HASH
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
)
endif()
#-----------------------------------------------------------------------------
# Configuration
#-----------------------------------------------------------------------------
set(BUILD_IMPORTER on CACHE BOOL "Build everything for importing/updating the database")
set(BUILD_API on CACHE BOOL "Build everything for the API server")
set(BUILD_TESTS on CACHE BOOL "Build test suite")
set(BUILD_OSM2PGSQL on CACHE BOOL "Build osm2pgsql (expert only)")
set(INSTALL_MUNIN_PLUGINS on CACHE BOOL "Install Munin plugins for supervising Nominatim")
#-----------------------------------------------------------------------------
# osm2pgsql (imports/updates only)
#-----------------------------------------------------------------------------
if (BUILD_IMPORTER AND BUILD_OSM2PGSQL)
if (NOT EXISTS "${CMAKE_SOURCE_DIR}/osm2pgsql/CMakeLists.txt")
message(FATAL_ERROR "The osm2pgsql directory is empty.\
Did you forget to check out Nominatim recursively?\
\nTry updating submodules with: git submodule update --init")
endif()
set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
set(BUILD_TESTS off)
add_subdirectory(osm2pgsql)
set(BUILD_TESTS ${BUILD_TESTS_SAVED})
endif()
#-----------------------------------------------------------------------------
# python (imports/updates only)
#-----------------------------------------------------------------------------
if (BUILD_IMPORTER OR BUILD_API)
find_package(PythonInterp 3.7 REQUIRED)
endif()
#-----------------------------------------------------------------------------
# import scripts and utilities (importer only)
#-----------------------------------------------------------------------------
if (BUILD_IMPORTER)
find_file(COUNTRY_GRID_FILE country_osm_grid.sql.gz
PATHS ${PROJECT_SOURCE_DIR}/data
NO_DEFAULT_PATH
DOC "Location of the country grid file."
)
if (NOT COUNTRY_GRID_FILE)
message(FATAL_ERROR "\nYou need to download the country_osm_grid first:\n"
" wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz")
endif()
configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl
${PROJECT_BINARY_DIR}/nominatim)
endif()
#-----------------------------------------------------------------------------
# Tests
#-----------------------------------------------------------------------------
if (BUILD_TESTS)
include(CTest)
set(TEST_BDD db osm2pgsql api)
find_program(PYTHON_BEHAVE behave)
find_program(PYLINT NAMES pylint3 pylint)
find_program(PYTEST NAMES pytest py.test-3 py.test)
if (PYTHON_BEHAVE)
message(STATUS "Using Python behave binary ${PYTHON_BEHAVE}")
foreach (test ${TEST_BDD})
add_test(NAME bdd_${test}
COMMAND ${PYTHON_BEHAVE} ${test}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/bdd)
set_tests_properties(bdd_${test}
PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
endforeach()
else()
message(WARNING "behave not found. BDD tests disabled." )
endif()
if (PYLINT)
message(STATUS "Using pylint binary ${PYLINT}")
add_test(NAME pylint
COMMAND ${PYLINT} nominatim
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
else()
message(WARNING "pylint not found. Python linting tests disabled.")
endif()
if (PYTEST)
message(STATUS "Using pytest binary ${PYTEST}")
add_test(NAME pytest
COMMAND ${PYTEST} test/python
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
else()
message(WARNING "pytest not found. Python tests disabled." )
endif()
endif()
#-----------------------------------------------------------------------------
# Installation
#-----------------------------------------------------------------------------
include(GNUInstallDirs)
set(NOMINATIM_DATADIR ${CMAKE_INSTALL_FULL_DATADIR}/${PROJECT_NAME})
set(NOMINATIM_LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME})
set(NOMINATIM_CONFIGDIR ${CMAKE_INSTALL_FULL_SYSCONFDIR}/${PROJECT_NAME})
set(NOMINATIM_MUNINDIR ${CMAKE_INSTALL_FULL_DATADIR}/munin/plugins)
if (BUILD_IMPORTER)
configure_file(${PROJECT_SOURCE_DIR}/cmake/tool-installed.tmpl installed.bin)
install(PROGRAMS ${PROJECT_BINARY_DIR}/installed.bin
DESTINATION ${CMAKE_INSTALL_BINDIR}
RENAME nominatim)
configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py-no-php.tmpl paths-py.installed)
foreach (submodule nominatim_db nominatim_api)
install(DIRECTORY src/${submodule}
DESTINATION ${NOMINATIM_LIBDIR}/lib-python
FILES_MATCHING PATTERN "*.py"
PATTERN "paths.py" EXCLUDE
PATTERN __pycache__ EXCLUDE)
install(FILES ${PROJECT_BINARY_DIR}/paths-py.installed
DESTINATION ${NOMINATIM_LIBDIR}/lib-python/${submodule}
RENAME paths.py)
endforeach()
install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR})
install(FILES ${COUNTRY_GRID_FILE}
data/words.sql
DESTINATION ${NOMINATIM_DATADIR})
endif()
if (BUILD_OSM2PGSQL)
if (${CMAKE_VERSION} VERSION_LESS 3.13)
# Installation of subdirectory targets was only introduced in 3.13.
# So just copy the osm2pgsql file for older versions.
install(PROGRAMS ${PROJECT_BINARY_DIR}/osm2pgsql/osm2pgsql
DESTINATION ${NOMINATIM_LIBDIR})
else()
install(TARGETS osm2pgsql RUNTIME DESTINATION ${NOMINATIM_LIBDIR})
endif()
endif()
install(FILES settings/env.defaults
settings/address-levels.json
settings/phrase-settings.json
settings/import-admin.lua
settings/import-street.lua
settings/import-address.lua
settings/import-full.lua
settings/import-extratags.lua
settings/flex-base.lua
settings/icu_tokenizer.yaml
settings/country_settings.yaml
DESTINATION ${NOMINATIM_CONFIGDIR})
install(DIRECTORY settings/icu-rules
DESTINATION ${NOMINATIM_CONFIGDIR})
install(DIRECTORY settings/country-names
DESTINATION ${NOMINATIM_CONFIGDIR})
if (INSTALL_MUNIN_PLUGINS)
install(FILES munin/nominatim_importlag
munin/nominatim_query_speed
munin/nominatim_requests
DESTINATION ${NOMINATIM_MUNINDIR})
endif()
message(WARNING "Building with CMake is deprecated and will be removed in Nominatim 5.0."
"Use Nominatim pip packages instead.\n"
"See https://nominatim.org/release-docs/develop/admin/Installation/#downloading-and-building-nominatim")

5
CONTRIBUTING.md
View File

@@ -92,13 +92,14 @@ Checklist for releases:
* [ ] complete `docs/admin/Migration.md` * [ ] complete `docs/admin/Migration.md`
* [ ] update EOL dates in `SECURITY.md` * [ ] update EOL dates in `SECURITY.md`
* [ ] commit and make sure CI tests pass * [ ] commit and make sure CI tests pass
* [ ] update OSMF production repo and release new version -post1 there
* [ ] test migration * [ ] test migration
* download, build and import previous version * download, build and import previous version
* migrate using master version * migrate using master version
* run updates using master version * run updates using master version
* [ ] prepare tarball: * [ ] prepare tarball:
* `git clone --recursive https://github.com/osm-search/Nominatim` (switch to right branch!) * `git clone https://github.com/osm-search/Nominatim` (switch to right branch!)
* `rm -r .git* osm2pgsql/.git*` * `rm -r .git*`
* copy country data into `data/` * copy country data into `data/`
* add version to base directory and package * add version to base directory and package
* [ ] upload tarball to https://nominatim.org * [ ] upload tarball to https://nominatim.org

16
README.md
View File

@@ -27,21 +27,21 @@ can be found at nominatim.org as well.
A quick summary of the necessary steps: A quick summary of the necessary steps:
1. Compile Nominatim: 1. Create a Python virtualenv and install the packages:
mkdir build python3 -m venv nominatim-venv
cd build ./nominatim-venv/bin/pip install packaging/nominatim-{api,db}
cmake ..
make
sudo make install
2. Create a project directory, get OSM data and import: 2. Create a project directory, get OSM data and import:
mkdir nominatim-project mkdir nominatim-project
cd nominatim-project cd nominatim-project
nominatim import --osm-file <your planet file> ../nominatim-venv/bin/nominatim import --osm-file <your planet file>
3. Point your webserver to the nominatim-project/website directory. 3. Start the webserver:
./nominatim-venv/bin/pip install uvicorn falcon
../nominatim-venv/bin/nominatim serve
License License

86
VAGRANT.md
View File

@@ -1,6 +1,6 @@
# Install Nominatim in a virtual machine for development and testing # Install Nominatim in a virtual machine for development and testing
This document describes how you can install Nominatim inside a Ubuntu 22 This document describes how you can install Nominatim inside a Ubuntu 24
virtual machine on your desktop/laptop (host machine). The goal is to give virtual machine on your desktop/laptop (host machine). The goal is to give
you a development environment to easily edit code and run the test suite you a development environment to easily edit code and run the test suite
without affecting the rest of your system. without affecting the rest of your system.
@@ -17,24 +17,17 @@ is.
3. Nominatim 3. Nominatim
git clone --recursive https://github.com/openstreetmap/Nominatim.git git clone https://github.com/openstreetmap/Nominatim.git
If you forgot `--recursive`, it you can later load the submodules using
git submodule init
git submodule update
## Installation ## Installation
1. Start the virtual machine 1. Start the virtual machine
vagrant up ubuntu vagrant up ubuntu24-nginx
2. Log into the virtual machine 2. Log into the virtual machine
vagrant ssh ubuntu vagrant ssh ubuntu24-nginx
3. Import a small country (Monaco) 3. Import a small country (Monaco)
@@ -61,73 +54,22 @@ see Nominatim in action on [localhost:8089](http://localhost:8089/nominatim/).
You edit code on your host machine in any editor you like. There is no need to You edit code on your host machine in any editor you like. There is no need to
restart any software: just refresh your browser window. restart any software: just refresh your browser window.
Note that the webserver uses files from the /build directory. If you change Use the functions of the `log()` object to create temporary debug output.
files in Nominatim/website or Nominatim/utils for example you first need to Add `&debug=1` to the URL to see the output.
copy them into the /build directory by running the `cmake` step from the
installation.
PHP errors are written to `/var/log/apache2/error.log`.
With `echo` and `var_dump()` you write into the output (HTML/XML/JSON) when
you either add `&debug=1` to the URL.
In the Python BDD test you can use `logger.info()` for temporary debug In the Python BDD test you can use `logger.info()` for temporary debug
statements. statements.
For more information on running tests, see
https://nominatim.org/release-docs/develop/develop/Testing/
## Running unit tests
cd ~/Nominatim/tests/php
phpunit ./
## Running PHP code style tests
cd ~/Nominatim
phpcs --colors .
## Running functional tests
Tests in `test/bdd/db` and `test/bdd/osm2pgsql` have to pass 100%. Other
tests might require full planet-wide data. Sadly even if you have your own
planet-wide data there will be enough differences to the openstreetmap.org
installation to cause false positives in the other tests (see FAQ).
To run the full test suite
cd ~/Nominatim/test/bdd
behave -DBUILDDIR=/home/vagrant/build/ db osm2pgsql
To run a single file
behave -DBUILDDIR=/home/vagrant/build/ api/lookup/simple.feature
Or a single test by line number
behave -DBUILDDIR=/home/vagrant/build/ api/lookup/simple.feature:34
To run specific groups of tests you can add tags just before the `Scenario line`, e.g.
@bug-34
Scenario: address lookup for non-existing or invalid node, way, relation
and then
behave -DBUILDDIR=/home/vagrant/build/ --tags @bug-34
## FAQ ## FAQ
##### Will it run on Windows? ##### Will it run on Windows?
Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need a 64bit Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need
version of Windows. a 64bit version of Windows.
##### Will it run on Apple Silicon? ##### Will it run on Apple Silicon?
@@ -136,11 +78,11 @@ There is no free/open source version of Parallels.
##### Why Monaco, can I use another country? ##### Why Monaco, can I use another country?
Of course! The Monaco import takes less than 30 minutes and works with 2GB RAM. Of course! The Monaco import takes less than 10 minutes and works with 2GB RAM.
##### Will the results be the same as those from nominatim.openstreetmap.org? ##### Will the results be the same as those from nominatim.openstreetmap.org?
No. Long running Nominatim installations will differ once new import features (or No. Long-running Nominatim installations will differ once new import features (or
bug fixes) get added since those usually only get applied to new/changed data. bug fixes) get added since those usually only get applied to new/changed data.
Also this document skips the optional Wikipedia data import which affects ranking Also this document skips the optional Wikipedia data import which affects ranking
@@ -188,7 +130,3 @@ e.g. `psql --host localhost --port 9999 nominatim_it`
Yes. It's possible to start the virtual machine on [Amazon AWS (plugin)](https://github.com/mitchellh/vagrant-aws) Yes. It's possible to start the virtual machine on [Amazon AWS (plugin)](https://github.com/mitchellh/vagrant-aws)
or [DigitalOcean (plugin)](https://github.com/smdahlen/vagrant-digitalocean). or [DigitalOcean (plugin)](https://github.com/smdahlen/vagrant-digitalocean).

14
cmake/paths-py-no-php.tmpl
View File

@@ -1,14 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Path settings for extra data used by Nominatim (installed version).
"""
from pathlib import Path
SQLLIB_DIR = (Path('@NOMINATIM_LIBDIR@') / 'lib-sql').resolve()
DATA_DIR = Path('@NOMINATIM_DATADIR@').resolve()
CONFIG_DIR = Path('@NOMINATIM_CONFIGDIR@').resolve()

13
cmake/tool-installed.tmpl
View File

@@ -1,13 +0,0 @@
#!/usr/bin/env python3
import sys
import os
sys.path.insert(1, '@NOMINATIM_LIBDIR@/lib-python')
from nominatim_db import cli
from nominatim_db import version
version.GIT_COMMIT_HASH = '@GIT_HASH@'
exit(cli.nominatim(module_dir='@NOMINATIM_LIBDIR@/module',
osm2pgsql_path='@NOMINATIM_LIBDIR@/osm2pgsql'))

13
cmake/tool.tmpl
View File

@@ -1,13 +0,0 @@
#!/usr/bin/env python3
import sys
import os
sys.path.insert(1, '@CMAKE_SOURCE_DIR@/src')
from nominatim_db import cli
from nominatim_db import version
version.GIT_COMMIT_HASH = '@GIT_HASH@'
exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',
osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql'))

62
docs/admin/Faq.md
View File

@@ -37,31 +37,6 @@ nominatim import --continue indexing
Otherwise it's best to start the full setup from the beginning. Otherwise it's best to start the full setup from the beginning.
### nominatim.so version mismatch
When running the import you may get a version mismatch:
`COPY_END for place failed: ERROR: incompatible library "/srv/Nominatim/nominatim/build/module/nominatim.so": version mismatch`
pg_config seems to use bad includes sometimes when multiple versions
of PostgreSQL are available in the system. Make sure you remove the
server development libraries (`postgresql-server-dev-13` on Ubuntu)
and recompile (`cmake .. && make`).
### I see the error "ERROR: permission denied for language c"
`nominatim.so`, written in C, is required to be installed on the database
server. Some managed database (cloud) services like Amazon RDS do not allow
this. There is currently no work-around other than installing a database
on a non-managed machine.
### I see the error: "function transliteration(text) does not exist"
Reinstall the nominatim functions with `nominatim refresh --functions`
and check for any errors, e.g. a missing `nominatim.so` file.
### I see the error: "ERROR: mmap (remap) failed" ### I see the error: "ERROR: mmap (remap) failed"
This may be a simple out-of-memory error. Try reducing the memory used This may be a simple out-of-memory error. Try reducing the memory used
@@ -103,39 +78,6 @@ for default Ubuntu operating system for example it's `www-data`.
GRANT SELECT ON ALL TABLES IN SCHEMA public TO "www-data"; GRANT SELECT ON ALL TABLES IN SCHEMA public TO "www-data";
``` ```
### Website reports "Could not load library "nominatim.so"
Example error message
```
SELECT make_standard_name('3039 E MEADOWLARK LN') [nativecode=ERROR: could not
load library "/srv/nominatim/Nominatim-3.1.0/build/module/nominatim.so":
/srv/nominatim/Nominatim-3.1.0/build/module/nominatim.so: cannot open shared
object file: Permission denied
CONTEXT: PL/pgSQL function make_standard_name(text) line 5 at assignment]
```
The PostgreSQL database, i.e. user `postgres`, needs to have access to that file.
The permission need to be read & executable by everybody, but not writeable
by everybody, e.g.
```
-rwxr-xr-x 1 nominatim nominatim 297984 build/module/nominatim.so
```
Try `chmod a+r nominatim.so; chmod a+x nominatim.so`.
When you recently updated your operating system, updated PostgreSQL to
a new version or moved files (e.g. the build directory) you should
recreate `nominatim.so`. Try
```
cd build
rm -r module/
cmake $main_Nominatim_path && make
```
### Setup fails with "DB Error: extension not found" ### Setup fails with "DB Error: extension not found"
Make sure you have the PostgreSQL extensions "hstore" and "postgis" installed. Make sure you have the PostgreSQL extensions "hstore" and "postgis" installed.
@@ -172,4 +114,6 @@ for more information.
### Can I import negative OSM ids into Nominatim? ### Can I import negative OSM ids into Nominatim?
See [this question of Stackoverflow](https://help.openstreetmap.org/questions/64662/nominatim-flatnode-with-negative-id). No, negative IDs are no longer supported by osm2pgsql. You can use
large 64-bit IDs that are guaranteed not to clash with OSM IDs. However,
you will not able to use a flatnode file with them.

51
docs/admin/Installation.md
View File

@@ -41,19 +41,6 @@ Furthermore the following Python libraries are required:
These will be installed automatically when using pip installation. These will be installed automatically when using pip installation.
When using legacy CMake-based installation:
* [cmake](https://cmake.org/)
* [expat](https://libexpat.github.io/)
* [proj](https://proj.org/)
* [bzip2](http://www.bzip.org/)
* [zlib](https://www.zlib.net/)
* [ICU](http://site.icu-project.org/)
* [nlohmann/json](https://json.nlohmann.me/)
* [Boost libraries](https://www.boost.org/), including system and file system
* PostgreSQL client libraries
* a recent C++ compiler (gcc 5+ or Clang 3.8+)
For running continuous updates: For running continuous updates:
* [pyosmium](https://osmcode.org/pyosmium/) * [pyosmium](https://osmcode.org/pyosmium/)
@@ -124,11 +111,10 @@ The release contains all necessary files. Just unpack it.
### Downloading the latest development version ### Downloading the latest development version
If you want to install latest development version from github, make sure to If you want to install latest development version from github:
also check out the osm2pgsql subproject:
``` ```
git clone --recursive https://github.com/openstreetmap/Nominatim.git git clone https://github.com/osm-search/Nominatim.git
``` ```
The development version does not include the country grid. Download it separately: The development version does not include the country grid. Download it separately:
@@ -139,8 +125,6 @@ wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/countr
### Building Nominatim ### Building Nominatim
#### Building the latest development version with pip
Nominatim is easiest to run from its own virtual environment. To create one, run: Nominatim is easiest to run from its own virtual environment. To create one, run:
sudo apt-get install virtualenv sudo apt-get install virtualenv
@@ -150,36 +134,5 @@ To install Nominatim directly from the source tree into the virtual environment,
/srv/nominatim-venv/bin/pip install packaging/nominatim-{db,api} /srv/nominatim-venv/bin/pip install packaging/nominatim-{db,api}
#### Building in legacy CMake mode
!!! warning
Installing Nominatim through CMake is now deprecated. The infrastructure
will be removed in Nominatim 5.0. Please switch to pip installation.
The code must be built in a separate directory. Create the directory and
change into it.
```
mkdir build
cd build
```
Nominatim uses cmake and make for building. Assuming that you have created the
build at the same level as the Nominatim source directory run:
```
cmake ../Nominatim
make
sudo make install
```
Nominatim installs itself into `/usr/local` per default. To choose a different
installation directory add `-DCMAKE_INSTALL_PREFIX=<install root>` to the
cmake command. Make sure that the `bin` directory is available in your path
in that case, e.g.
```
export PATH=<install root>/bin:$PATH
```
Now continue with [importing the database](Import.md). Now continue with [importing the database](Import.md).

1
osm2pgsql

Submodule osm2pgsql deleted from cf66989fa2

59
vagrant/Install-on-Ubuntu-22.sh
View File

@@ -25,15 +25,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \ libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
nlohmann-json3-dev postgresql-14-postgis-3 \ nlohmann-json3-dev postgresql-14-postgis-3 \
postgresql-contrib-14 postgresql-14-postgis-3-scripts \ postgresql-contrib-14 postgresql-14-postgis-3-scripts \
libicu-dev python3-dotenv \ libicu-dev virtualenv git
python3-pip python3-psutil python3-jinja2 \
python3-sqlalchemy python3-asyncpg \
python3-icu python3-datrie python3-yaml git
# Some of the Python packages that come with Ubuntu 22.04 are too old,
# so install the latest version from pip:
pip3 install --user psycopg[binary]
# #
# System Configuration # System Configuration
@@ -109,7 +101,7 @@ fi #DOCS:
# #
if [ "x$1" == "xyes" ]; then #DOCS: :::sh if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME cd $USERHOME
git clone --recursive https://github.com/openstreetmap/Nominatim.git git clone https://github.com/osm-search/Nominatim.git
cd Nominatim cd Nominatim
else #DOCS: else #DOCS:
cd $USERHOME/Nominatim #DOCS: cd $USERHOME/Nominatim #DOCS:
@@ -122,14 +114,31 @@ if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
fi #DOCS: fi #DOCS:
# The code must be built in a separate directory. Create this directory, # Nominatim needs osm2pgsql >= 1.8. The version that comes with Ubuntu is
# then configure and build Nominatim in there: # too old. Download and compile your own:
mkdir $USERHOME/build cd $USERHOME
cd $USERHOME/build git clone https://github.com/osm2pgsql-dev/osm2pgsql
cmake $USERHOME/Nominatim mkdir osm2pgsql-build
cd osm2pgsql-build
cmake ../osm2pgsql
make make
sudo make install sudo make install
cd $USERHOME/Nominatim
# Nominatim should be installed in a separate Python virtual environment.
# Create the virtual environment:
virtualenv $USERHOME/nominatim-venv
# We want the faster binary version pf psycopg, so install that:
$USERHOME/nominatim-venv/bin/pip install psycopg[binary]
# Now install Nominatim using pip:
cd $USERHOME/Nominatim
$USERHOME/nominatim-venv/bin/pip install packaging/nominatim-db
# Nominatim is now ready to use. You can continue with # Nominatim is now ready to use. You can continue with
# [importing a database from OSM data](../admin/Import.md). If you want to set up # [importing a database from OSM data](../admin/Import.md). If you want to set up
@@ -138,18 +147,19 @@ fi #DOCS:
# Setting up the Python frontend # Setting up the Python frontend
# ============================== # ==============================
# #
# Some of the Python packages in Ubuntu are too old. Therefore run the # The Python frontend is contained in the nominatim-api package. To run
# frontend from a Python virtualenv with current packages. # the API as a webservice, you also need falcon with uvicorn to serve the API.
# It is generally recommended to run falcon/uvicorn on top of gunicorn.
# #
# To set up the virtualenv, run: # To install all packages, run:
#DOCS:```sh #DOCS:```sh
sudo apt-get install -y virtualenv $USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn
virtualenv $USERHOME/nominatim-venv cd $USERHOME/Nominatim
$USERHOME/nominatim-venv/bin/pip install SQLAlchemy PyICU psycopg[binary] \ $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api
psycopg2-binary python-dotenv PyYAML falcon uvicorn gunicorn
#DOCS:``` #DOCS:```
# Next you need to create a systemd job that runs Nominatim on gunicorn. # Next you need to create a systemd job that runs Nominatim on gunicorn.
# First create a systemd job that manages the socket file: # First create a systemd job that manages the socket file:
@@ -178,14 +188,11 @@ Requires=nominatim.socket
[Service] [Service]
Type=simple Type=simple
Environment="PYTHONPATH=/usr/local/lib/nominatim/lib-python/"
User=www-data User=www-data
Group=www-data Group=www-data
WorkingDirectory=$USERHOME/nominatim-project WorkingDirectory=$USERHOME/nominatim-project
ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker "nominatim_api.server.falcon.server:run_wsgi()"
ExecReload=/bin/kill -s HUP \$MAINPID ExecReload=/bin/kill -s HUP \$MAINPID
StandardOutput=append:/var/log/gunicorn-nominatim.log
StandardError=inherit
PrivateTmp=true PrivateTmp=true
TimeoutStopSec=5 TimeoutStopSec=5
KillMode=mixed KillMode=mixed

12
vagrant/Install-on-Ubuntu-24.sh
View File

@@ -98,7 +98,7 @@ fi #DOCS:
# #
if [ "x$1" == "xyes" ]; then #DOCS: :::sh if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME cd $USERHOME
git clone https://github.com/openstreetmap/Nominatim.git git clone https://github.com/osm-search/Nominatim.git
cd Nominatim cd Nominatim
else #DOCS: else #DOCS:
cd $USERHOME/Nominatim #DOCS: cd $USERHOME/Nominatim #DOCS:
@@ -116,6 +116,10 @@ fi #DOCS:
virtualenv $USERHOME/nominatim-venv virtualenv $USERHOME/nominatim-venv
# We want the faster binary version pf psycopg, so install that:
$USERHOME/nominatim-venv/bin/pip install psycopg[binary]
# Now install Nominatim using pip: # Now install Nominatim using pip:
cd $USERHOME/Nominatim cd $USERHOME/Nominatim
@@ -143,7 +147,7 @@ fi #DOCS:
# To install all packages, run: # To install all packages, run:
#DOCS:```sh #DOCS:```sh
$USERHOME/nominatim-venv/bin/pip install psycopg[binary] falcon uvicorn gunicorn $USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn
cd $USERHOME/Nominatim cd $USERHOME/Nominatim
$USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api
#DOCS:``` #DOCS:```
@@ -179,10 +183,8 @@ Type=simple
User=www-data User=www-data
Group=www-data Group=www-data
WorkingDirectory=$USERHOME/nominatim-project WorkingDirectory=$USERHOME/nominatim-project
ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi ExecStart=$USERHOME/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker "nominatim_api.server.falcon.server:run_wsgi()"
ExecReload=/bin/kill -s HUP \$MAINPID ExecReload=/bin/kill -s HUP \$MAINPID
StandardOutput=append:/var/log/gunicorn-nominatim.log
StandardError=inherit
PrivateTmp=true PrivateTmp=true
TimeoutStopSec=5 TimeoutStopSec=5
KillMode=mixed KillMode=mixed