hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-14 18:37:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1957 from lonvia/docs-separate-out-deployment

Browse Source 
Restructure vagrant scripts and installation documentation
This commit is contained in:
Sarah Hoffmann
2020-09-17 21:22:48 +02:00
committed by GitHub
parent b8ced5d96b dbe025fe40
commit 3aa6e6a365
15 changed files with 738 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@@ -146,6 +146,11 @@ if (BUILD_API)
php -S 127.0.0.1:8088
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
)
add_custom_target(serve-global
php -S 0.0.0.0:8088
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
)
endif()
#-----------------------------------------------------------------------------

91
Vagrantfile vendored
View File

@@ -4,18 +4,38 @@
Vagrant.configure("2") do |config|
# Apache webserver
config.vm.network "forwarded_port", guest: 80, host: 8089
config.vm.network "forwarded_port", guest: 8088, host: 8088
# If true, then any SSH connections made will enable agent forwarding.
config.ssh.forward_agent = true
# Never sync the current directory to /vagrant.
config.vm.synced_folder ".", "/vagrant", disabled: true
checkout = "yes"
if ENV['CHECKOUT'] != 'y' then
config.vm.synced_folder ".", "/home/vagrant/Nominatim"
checkout = "no"
checkout = "no"
end
config.vm.provider "virtualbox" do |vb, override|
vb.gui = false
vb.memory = 2048
vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate//vagrant","0"]
if ENV['CHECKOUT'] != 'y' then
override.vm.synced_folder ".", "/home/vagrant/Nominatim"
end
end
config.vm.provider "libvirt" do |lv, override|
lv.memory = 2048
lv.nested = true
if ENV['CHECKOUT'] != 'y' then
override.vm.synced_folder ".", "/home/vagrant/Nominatim", type: 'nfs'
end
end
config.vm.define "ubuntu", primary: true do |sub|
sub.vm.box = "bento/ubuntu-20.04"
sub.vm.box = "generic/ubuntu2004"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Ubuntu-20.sh"
s.privileged = false
@@ -23,8 +43,26 @@ Vagrant.configure("2") do |config|
end
end
config.vm.define "ubuntu18", primary: true do |sub|
sub.vm.box = "bento/ubuntu-18.04"
config.vm.define "ubuntu-apache" do |sub|
sub.vm.box = "generic/ubuntu2004"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Ubuntu-20.sh"
s.privileged = false
s.args = [checkout, "install-apache"]
end
end
config.vm.define "ubuntu-nginx" do |sub|
sub.vm.box = "generic/ubuntu2004"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Ubuntu-20.sh"
s.privileged = false
s.args = [checkout, "install-nginx"]
end
end
config.vm.define "ubuntu18" do |sub|
sub.vm.box = "generic/ubuntu1804"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Ubuntu-18.sh"
s.privileged = false
@@ -32,60 +70,41 @@ Vagrant.configure("2") do |config|
end
end
config.vm.define "ubuntu18nginx" do |sub|
sub.vm.box = "bento/ubuntu-18.04"
config.vm.define "ubuntu18-apache" do |sub|
sub.vm.box = "generic/ubuntu1804"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Ubuntu-18-nginx.sh"
s.path = "vagrant/Install-on-Ubuntu-18.sh"
s.privileged = false
s.args = [checkout]
s.args = [checkout, "install-apache"]
end
end
config.vm.define "ubuntu16" do |sub|
sub.vm.box = "bento/ubuntu-16.04"
config.vm.define "ubuntu18-nginx" do |sub|
sub.vm.box = "generic/ubuntu1804"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Ubuntu-16.sh"
s.path = "vagrant/Install-on-Ubuntu-18.sh"
s.privileged = false
s.args = [checkout]
s.args = [checkout, "install-nginx"]
end
end
config.vm.define "travis" do |sub|
sub.vm.box = "bento/ubuntu-14.04"
config.vm.define "centos7" do |sub|
sub.vm.box = "centos/7"
sub.vm.provision :shell do |s|
s.path = "vagrant/install-on-travis-ci.sh"
s.path = "vagrant/Install-on-Centos-7.sh"
s.privileged = false
s.args = [checkout]
end
end
config.vm.define "centos" do |sub|
sub.vm.box = "centos/7"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Centos-7.sh"
s.privileged = false
s.args = "yes"
end
sub.vm.synced_folder ".", "/home/vagrant/Nominatim", disabled: true
sub.vm.synced_folder ".", "/vagrant", disabled: true
end
config.vm.define "centos8" do |sub|
sub.vm.box = "generic/centos8"
sub.vm.provision :shell do |s|
s.path = "vagrant/Install-on-Centos-8.sh"
s.privileged = false
s.args = "yes"
s.args = [checkout]
end
sub.vm.synced_folder ".", "/home/vagrant/Nominatim", disabled: true
sub.vm.synced_folder ".", "/vagrant", disabled: true
end
config.vm.provider "virtualbox" do |vb|
vb.gui = false
vb.memory = 2048
vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate//vagrant","0"]
end
end

141
docs/admin/Deployment.md Normal file
View File

@@ -0,0 +1,141 @@
# Deploying Nominatim
The Nominatim API is implemented as a PHP application. The `website/` directory
in the build directory contains the configured website. You can serve this
in a production environment with any web server that is capable to run
PHP scripts.
This section gives a quick overview on how to configure Apache and Nginx to
serve Nominatim. It is not meant as a full system administration guide on how
to run a web service. Please refer to the documentation of
[Apache](http://httpd.apache.org/docs/current/) and
[Nginx](https://nginx.org/en/docs/)
for background information on configuring the services.
!!! Note
Throughout this page, we assume that your Nominatim build directory is
located in `/srv/nominatim/build` and the source code in
`/srv/nominatim/Nominatim`. If you have put it somewhere else, you
need to adjust the commands and configuration accordingly.
We further assume that your web server runs as user `www-data`. Older
versions of CentOS may still use the user name `apache`. You also need
to adapt the instructions in this case.
## Making the website directory accessible
You need to make sure that the `website` directory is accessible for the
web server user. You can check that the permissions are correct by accessing
on of the php files as the web server user:
``` sh
sudo -u www-data head -n 1 /srv/nominatim/build/website/search.php
```
If this shows a permission error, then you need to adapt the permissions of
each directory in the path so that it is executable for `www-data`.
If you have SELinux enabled, further adjustments may be necessary to give the
web server access. At a minimum the following SELinux labelling should be done
for Nominatim:
``` sh
sudo semanage fcontext -a -t httpd_sys_content_t "/srv/nominatim/Nominatim/(website|lib|settings)(/.*)?"
sudo semanage fcontext -a -t httpd_sys_content_t "/srv/nominatim/build/(website|settings)(/.*)?"
sudo semanage fcontext -a -t lib_t "/srv/nominatim/build/module/nominatim.so"
sudo restorecon -R -v /srv/nominatim/Nominatim
sudo restorecon -R -v /srv/nominatim/build
```
## Nominatim with Apache
### Installing the required packages
With Apache you can use the PHP module to run Nominatim.
Under Ubuntu/Debian install them with:
``` sh
sudo apt install apache2 libapache2-mod-php
```
### Configuring Apache
Make sure your Apache configuration contains the required permissions for the
directory and create an alias:
``` apache
<Directory "/srv/nominatim/build/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim /srv/nominatim/build/website
```
After making changes in the apache config you need to restart apache.
The website should now be available on `http://localhost/nominatim`.
## Nominatim with Nginx
### Installing the required packages
Nginx has no built-in PHP interpreter. You need to use php-fpm as a deamon for
serving PHP cgi.
On Ubuntu/Debian install nginx and php-fpm with:
``` sh
sudo apt install nginx php-fpm
```
### Configure php-fpm and Nginx
By default php-fpm listens on a network socket. If you want it to listen to a
Unix socket instead, change the pool configuration
(`/etc/php/<php version>/fpm/pool.d/www.conf`) as follows:
``` ini
; Replace the tcp listener and add the unix socket
listen = /var/run/php-fpm.sock
; Ensure that the daemon runs as the correct user
listen.owner = www-data
listen.group = www-data
listen.mode = 0666
```
Tell nginx that php files are special and to fastcgi_pass to the php-fpm
unix socket by adding the location definition to the default configuration.
``` nginx
root /srv/nominatim/build/website;
index search.php;
location / {
try_files $uri $uri/ @php;
}
location @php {
fastcgi_param SCRIPT_FILENAME "$document_root$uri.php";
fastcgi_param PATH_TRANSLATED "$document_root$uri.php";
fastcgi_param QUERY_STRING $args;
fastcgi_pass unix:/var/run/php-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
}
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f $document_root$fastcgi_script_name) {
return 404;
}
fastcgi_pass unix:/var/run/php-fpm.sock;
fastcgi_index search.php;
include fastcgi.conf;
}
```
Restart the nginx and php-fpm services and the website should now be available
at `http://localhost/`.

26
docs/admin/Import.md
View File

@@ -187,7 +187,7 @@ enough RAM for PostgreSQL and osm2pgsql as mentioned above. If the system starts
swapping or you are getting out-of-memory errors, reduce the cache size or
even consider using a flatnode file.
### Verify import finished
### Verify the import
Run this script to verify all required tables and indices got created successfully.
@@ -197,9 +197,8 @@ Run this script to verify all required tables and indices got created successful
### Setting up the website
Run the following command to set up the `website/settings-frontend.php`.
These settings are used in website/*.php files. You can use the website only after this
step is completed.
Run the following command to set up the configuration file for the website
`settings/settings-frontend.php`. These settings are used in website/*.php files.
```sh
./utils/setup.php --setup-website
@@ -207,6 +206,20 @@ step is completed.
!!! Note
This step is not necessary if you use `--all` option while setting up the DB.
Now you can try out your installation by running:
```sh
make serve
```
This runs a small test server normally used for development. You can use it
to verify that your installation is working. Go to
`http://localhost:8088/status.php` and you should see the message `OK`.
You can also run a search query, e.g. `http://localhost:8088/search.php?q=Berlin`.
To run Nominatim via webservers like Apache or nginx, please read the
[Deployment chapter](Deployment.md).
## Tuning the database
Accurate word frequency information for search terms helps PostgreSQL's query
@@ -247,8 +260,6 @@ entire US adds about 10GB to your database.
wget https://nominatim.org/data/tiger2019-nominatim-preprocessed.tar.gz
tar xf tiger2019-nominatim-preprocessed.tar.gz
`data-source/overview.md` explains how the data got preprocessed.
2. Import the data into your Nominatim database:
./utils/setup.php --import-tiger-data
@@ -264,3 +275,6 @@ entire US adds about 10GB to your database.
```
See the [developer's guide](../develop/data-sources.md#us-census-tiger) for more
information on how the data got preprocessed.

108
docs/admin/Installation.md
View File

@@ -43,17 +43,13 @@ For running Nominatim:
* [PHP](https://php.net) (7.0 or later)
* PHP-pgsql
* PHP-intl (bundled with PHP)
* a webserver (apache or nginx are recommended)
For running continuous updates:
* [pyosmium](https://osmcode.org/pyosmium/) (with Python 3)
For running tests:
* [behave](http://pythonhosted.org/behave/)
* [nose](https://nose.readthedocs.io)
* [phpunit](https://phpunit.de) >= 7.3
For dependencies for running tests and building documentation, see
the [Development section](../develop/Development-Environment.md).
### Hardware
@@ -68,9 +64,7 @@ will help considerably to speed up import and queries.
Even on a well configured machine the import of a full planet takes
at least 2 days. Without SSDs 7-8 days are more realistic.
## Setup of the server
### PostgreSQL tuning
## Tuning the PostgreSQL database
You might want to tune your PostgreSQL installation so that the later steps
make best use of your hardware. You should tune the following parameters in
@@ -110,82 +104,44 @@ Don't forget to reenable them after the initial import or you risk database
corruption.
### Webserver setup
## Downloading and building Nominatim
The `website/` directory in the build directory contains the configured
website. Include the directory into your webbrowser to serve php files
from there.
### Downloading the latest release
#### Configure for use with Apache
You can download the [latest release from nominatim.org](https://nominatim.org/downloads/).
The release contains all necessary files. Just unpack it.
Make sure your Apache configuration contains the required permissions for the
directory and create an alias:
### Downloading the latest development version
``` apache
<Directory "/srv/nominatim/build/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim /srv/nominatim/build/website
If you want to install latest development version from github, make sure to
also check out the osm2pgsql subproject:
```
git clone --recursive git://github.com/openstreetmap/Nominatim.git
```
`/srv/nominatim/build` should be replaced with the location of your
build directory.
The development version does not include the country grid. Download it separately:
After making changes in the apache config you need to restart apache.
The website should now be available on http://localhost/nominatim.
#### Configure for use with Nginx
Use php-fpm as a deamon for serving PHP cgi. Install php-fpm together with nginx.
By default php listens on a network socket. If you want it to listen to a
Unix socket instead, change the pool configuration (`pool.d/www.conf`) as
follows:
; Comment out the tcp listener and add the unix socket
;listen = 127.0.0.1:9000
listen = /var/run/php5-fpm.sock
; Ensure that the daemon runs as the correct user
listen.owner = www-data
listen.group = www-data
listen.mode = 0666
Tell nginx that php files are special and to fastcgi_pass to the php-fpm
unix socket by adding the location definition to the default configuration.
``` nginx
root /srv/nominatim/build/website;
index search.php;
location / {
try_files $uri $uri/ @php;
}
location @php {
fastcgi_param SCRIPT_FILENAME "$document_root$uri.php";
fastcgi_param PATH_TRANSLATED "$document_root$uri.php";
fastcgi_param QUERY_STRING $args;
fastcgi_pass unix:/var/run/php/php7.3-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
}
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f $document_root$fastcgi_script_name) {
return 404;
}
fastcgi_pass unix:/var/run/php7.3-fpm.sock;
fastcgi_index search.php;
include fastcgi.conf;
}
```
wget -O Nominatim/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
```
Restart the nginx and php5-fpm services and the website should now be available
at `http://localhost/`.
### Building Nominatim
The code must be built in a separate directory. Create the directory and
change into it.
```
mkdir build
cd build
```
Nominatim uses cmake and make for building. Assuming that you have created the
build at the same level as the Nominatim source directory run:
```
cmake ../Nominatim
make
```
Now continue with [importing the database](Import.md).

170
docs/develop/Development-Environment.md Normal file
View File

@@ -0,0 +1,170 @@
# Setting up Nominatim for Development
This chapter gives an overview how to set up Nominatim for developement
and how to run tests.
!!! Important
This guide assumes that you develop under the latest version of Ubuntu. You
can of course also use your favourite distribution. You just might have to
adapt the commands below slightly, in particular the commands for installing
additional software.
## Installing Nominatim
The first step is to install Nominatim itself. Please follow the installation
instructions in the [Admin section](../admin/Installation.md). You don't need
to set up a webserver for development, the webserver that is included with PHP
is sufficient.
If you want to run Nominatim in a VM via Vagrant, use the default `ubuntu` setup.
Vagrant's libvirt provider runs out-of-the-box under Ubuntu. You also need to
install an NFS daemon to enable directory sharing between host and guest. The
following packages should get you started:
sudo apt install vagrant vagrant-libvirt libvirt-daemon nfs-kernel-server
## Prerequisites for testing and documentation
The Nominatim tests suite consists of behavioural tests (using behave) and
unit tests (using PHPUnit). It has the following additional requirements:
* [behave test framework](https://github.com/behave/behave) >= 1.2.5
* [nose](https://nose.readthedocs.org)
* [pytidylib](http://countergram.com/open-source/pytidylib)
* [phpunit](https://phpunit.de) >= 7.3
* [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer)
The documentation is built with mkdocs:
* [mkdocs](https://www.mkdocs.org/) >= 1.1.2
### Installing prerequisites on Ubuntu/Debian
Some of the Python packages require the newest version which is not yet
available with the current distributions. Therefore it is recommended to
install pip to get the newest versions.
To install all necessary packages run:
```sh
sudo apt install php-cgi phpunit php-codesniffer \
python3-pip python3-setuptools python3-dev
pip3 install --user behave nose mkdocs
```
The `mkdocs` executable will be located in `.local/bin`. You may have to add
this directory to your path, for example by running:
```
echo 'export PATH=~/.local/bin:$PATH' > ~/.profile
```
If your distribution does not have PHPUnit 7.3+, you can install it (as well
as CodeSniffer) via composer:
```
sudo apt-get install composer
composer global require "squizlabs/php_codesniffer=*"
composer global require "phpunit/phpunit=8.*"
```
The binaries are found in `.config/composer/vendor/bin`. You need to add this
to your PATH as well:
```
echo 'export PATH=~/.config/composer/vendor/bin:$PATH' > ~/.profile
```
## Executing Tests
All tests are located in the `\test` directory.
### Preparing the test database
Some of the behavioural test expect a test database to be present. You need at
least 2GB RAM and 10GB disk space to create the database.
First create a separate directory for the test DB and Fetch the test planet
data and the Tiger data for South Dakota:
```
mkdir testdb
cd testdb
wget https://www.nominatim.org/data/test/nominatim-api-testdata.pbf
wget -O - https://nominatim.org/data/tiger2018-nominatim-preprocessed.tar.gz | tar xz --wildcards --no-anchored '46*'
```
Configure and build Nominatim in the usual way:
```
cmake $USERNAME/Nominatim
make
```
Copy the test settings:
```
cp $USERNAME/Nominatim/test/testdb/local.php settings/
```
Inspect the file to check that all settings are correct for your local setup.
Now you can import the test database:
```
dropdb --if-exists test_api_nominatim
./utils/setup.php --all --osm-file nominatim-api-testdb.pbf 2>&1 | tee import.log
./utils/specialphrases.php --wiki-import | psql -d test_api_nominatim 2>&1 | tee -a import.log
./utils/setup.php --import-tiger-data 2>&1 | tee -a import.log
```
### Running the tests
To run all tests just go to the test directory and run make:
```sh
cd test
make
```
To skip tests that require the test database, run `make no-test-db` instead.
For more information about the structure of the tests and how to change and
extend the test suite, see the [Testing chapter](Testing.md).
## Documentation Pages
The [Nominatim documentation](https://nominatim.org/release-docs/develop/) is
built using the [MkDocs](https://www.mkdocs.org/) static site generation
framework. The master branch is automatically deployed every night on
[https://nominatim.org/release-docs/develop/](https://nominatim.org/release-docs/develop/)
To build the documentation, go to the build directory and run
```
make doc
INFO - Cleaning site directory
INFO - Building documentation to directory: /home/vagrant/build/site-html
```
This runs `mkdocs build` plus extra transformation of some files and adds
symlinks (see `CMakeLists.txt` for the exact steps).
Now you can start webserver for local testing
```
build> mkdocs serve
[server:296] Serving on http://127.0.0.1:8000
[handlers:62] Start watching changes
```
If you develop inside a Vagrant virtual machine, use a port that is forwarded
to your host:
```
build> mkdocs serve --dev-addr 0.0.0.0:8088
[server:296] Serving on http://0.0.0.0:8088
[handlers:62] Start watching changes
```

39
docs/develop/Documentation.md
View File

@@ -1,39 +0,0 @@
# Documentation Pages
The [Nominatim documentation](https://nominatim.org/release-docs/develop/) is built using the [MkDocs](https://www.mkdocs.org/) static site generation framework. The master branch is automatically deployed every night on under [https://nominatim.org/release-docs/develop/](https://nominatim.org/release-docs/develop/)
To preview local changes, first install MkDocs
```
pip3 install --user mkdocs
```
If `mkdocs` can't be found after the installation, the $PATH might have not
been set correctly yet. Try opening a new terminal session.
Then go to the build directory and run
```
make doc
INFO - Cleaning site directory
INFO - Building documentation to directory: /home/vagrant/build/site-html
```
This runs `mkdocs build` plus extra transformation of some files and adds
symlinks (see `CMakeLists.txt` for the exact steps).
Now you can start webserver for local testing
```
build> mkdocs serve
[server:296] Serving on http://127.0.0.1:8000
[handlers:62] Start watching changes
```
If you develop inside a Vagrant virtual machine:
* add port forwarding to your Vagrantfile,
e.g. `config.vm.network "forwarded_port", guest: 8000, host: 8000`
* use `mkdocs serve --dev-addr 0.0.0.0:8000` because the default localhost
IP does not get forwarded.

52
docs/develop/Setup.md
View File

@@ -1,52 +0,0 @@
# Setup Test Environment
To test changes and contribute to Nominatim you should be able to run
the test suite(s). For many usecases it's enough to create a Vagrant
virtual machine (see `VAGRANT.md`), import one small country into the
database.
## Prerequisites
Nominatim supports a range of PHP versions and PHPUnit versions also
move fast. We try to test against the newest stable PHP release and
PHPUnit version even though we expect many Nominatim users will install
older version on their production servers.
* Python 3 (https://www.python.org/)
* behave test framework >= 1.2.5 (https://github.com/behave/behave)
* nose (https://nose.readthedocs.org)
* pytidylib (http://countergram.com/open-source/pytidylib)
* psycopg2 (http://initd.org/psycopg/)
#### Ubuntu 20
sudo apt-get install -y phpunit php-codesniffer php-cgi
pip3 install --user behave nose
#### Ubuntu 18
pip3 install --user behave nose
sudo apt-get install -y composer php-cgi php-cli php-mbstring php-xml zip unzip
composer global require "squizlabs/php_codesniffer=*"
sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/
composer global require "phpunit/phpunit=8.*"
sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/
#### CentOS 7 or 8
sudo dnf install -y php-dom php-mbstring
pip3 install --user behave nose
composer global require "squizlabs/php_codesniffer=*"
sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/
composer global require "phpunit/phpunit=^7"
sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/
## Run tests, code linter, code coverage
See `README.md` in `test` subdirectory.

110
test/README.md → docs/develop/Testing.md
View File

@@ -1,12 +1,10 @@
This directory contains functional and unit tests for the Nominatim API.
# Nominatim Test Suite
Prerequisites
=============
This chapter describes the tests in the `/test` directory, how they are
structured and how to extend them. For a quick introduction on how to run
the tests, see the [Development setup chapter](Development-Environment.md).
See `docs/develop/Setup.md`
Overall structure
=================
## Overall structure
There are two kind of tests in this test suite. There are functional tests
which test the API interface using a BDD test framework and there are unit
@@ -27,8 +25,7 @@ This test directory is sturctured as follows:
+- testdb Base data for generating API test database
```
PHP Unit Tests
==============
## PHP Unit Tests (`test/php`)
Unit tests can be found in the php/ directory and tests selected php functions.
Very low coverage.
@@ -44,14 +41,19 @@ strip and set other parameters.
It will use (and destroy) a local database 'nominatim_unit_tests'. You can set
a different connection string with e.g. UNIT_TEST_DSN='pgsql:dbname=foo_unit_tests'.
BDD Functional Tests
====================
## BDD Functional Tests (`test/bdd`)
Functional tests are written as BDD instructions. For more information on
the philosophy of BDD testing, see http://pythonhosted.org/behave/philosophy.html
the philosophy of BDD testing, see the
[Behave manual](http://pythonhosted.org/behave/philosophy.html).
Usage
-----
The following explanation assume that the reader is familiar with the BDD
notations of features, scenarios and steps.
All possible steps can be found in the `steps` directory and should ideally
be documented.
### General Usage
To run the functional tests, do
@@ -81,62 +83,29 @@ The tests can be configured with a set of environment variables (`behave -D key=
run, otherwise the result is undefined.
Logging can be defined through command line parameters of behave itself. Check
out `behave --help` for details. Also keep an eye out for the 'work-in-progress'
out `behave --help` for details. Also have a look at the 'work-in-progress'
feature of behave which comes in handy when writing new tests.
Writing Tests
-------------
The following explanation assume that the reader is familiar with the BDD
notations of features, scenarios and steps.
All possible steps can be found in the `steps` directory and should ideally
be documented.
### API Tests (`test/bdd/api`)
These tests are meant to test the different API endpoints and their parameters.
They require a to import several datasets into a test database. You need at
least 2GB RAM and 10GB discspace.
They require a to import several datasets into a test database.
See the [Development Setup chapter](Development-Environment.md#preparing-the-test-database)
for instructions on how to set up this database.
The official test dataset was derived from the 180924 planet (note: such
file no longer exists at https://planet.openstreetmap.org/planet/2018/).
Newer planets are likely to work as well but you may see isolated test
failures where the data has changed.
1. Fetch the OSM planet extract (200MB). See end of document how it got created.
The official test dataset can always be downloaded from
[nominatim.org](https://www.nominatim.org/data/test/nominatim-api-testdata.pbf)
To recreate the input data for the test database run:
```
cd Nominatim/data
mkdir testdb
cd testdb
wget https://www.nominatim.org/data/test/nominatim-api-testdata.pbf
```
2. Fetch `46*` (South Dakota) Tiger data
```
cd Nominatim/data/testdb
wget https://nominatim.org/data/tiger2018-nominatim-preprocessed.tar.gz
tar xvf tiger2018-nominatim-preprocessed.tar.gz --wildcards --no-anchored '46*'
rm tiger2018-nominatim-preprocessed.tar.gz
```
3. Adapt build/settings/local.php settings:
```
@define('CONST_Database_DSN', 'pgsql:dbname=test_api_nominatim');
@define('CONST_Use_US_Tiger_Data', true);
@define('CONST_Tiger_Data_Path', CONST_ExtraDataPath.'/testdb');
@define('CONST_Wikipedia_Data_Path', CONST_BasePath.'/test/testdb');
```
4. Import
```
LOGFILE=/tmp/nominatim-testdb.$$.log
dropdb --if-exists test_api_nominatim
./utils/setup.php --all --osm-file ../Nominatim/data/testdb/nominatim-api-testdb.pbf 2>&1 | tee $LOGFILE
./utils/specialphrases.php --wiki-import > specialphrases.sql
psql -d test_api_nominatim -f specialphrases.sql 2>&1 | tee -a $LOGFILE
./utils/setup.php --import-tiger-data 2>&1 | tee -a $LOGFILE
```
```
wget https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-180924.osm.pbf
osmconvert planet-180924.osm.pbf -B=test/testdb/testdb.polys -o=testdb.pbf
```
#### Code Coverage
@@ -146,7 +115,7 @@ On Debian/Ubuntu run:
apt-get install php-codecoverage php-xdebug
The run the API tests as follows:
Then run the API tests as follows:
behave api -DPHPCOV=<coverage output dir>
@@ -155,7 +124,7 @@ the [phpcov](https://github.com/sebastianbergmann/phpcov) tool:
phpcov merge --html=<report output dir> <coverage output dir>
### Indexing Tests (`test/bdd/db`)
### DB Creation Tests (`test/bdd/db`)
These tests check the import and update of the Nominatim database. They do not
test the correctness of osm2pgsql. Each test will write some data into the `place`
@@ -171,16 +140,3 @@ needs superuser rights for postgres.
These tests check that data is imported correctly into the place table. They
use the same template database as the Indexing tests, so the same remarks apply.
How the test database extract was generated
-------------------------------------------
The official test dataset was derived from the 180924 planet (note: such
file no longer exists at https://planet.openstreetmap.org/planet/2018/).
Newer planets are likely to work as well but you may see isolated test
failures where the data has changed. To recreate the input data
for the test database run:
wget https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-180924.osm.pbf
osmconvert planet-180924.osm.pbf -B=test/testdb/testdb.polys -o=testdb.pbf

11
docs/mkdocs.yml
View File

@@ -16,19 +16,20 @@ pages:
- 'FAQ': 'api/Faq.md'
- 'Administration Guide':
- 'Basic Installation': 'admin/Installation.md'
- 'Importing' : 'admin/Import.md'
- 'Updating' : 'admin/Update.md'
- 'Import' : 'admin/Import.md'
- 'Update' : 'admin/Update.md'
- 'Deploy' : 'admin/Deployment.md'
- 'Nominatim UI' : 'admin/Setup-Nominatim-UI.md'
- 'Advanced Installations' : 'admin/Advanced-Installations.md'
- 'Migration from older Versions' : 'admin/Migration.md'
- 'Troubleshooting' : 'admin/Faq.md'
- 'Developers Guide':
- 'Overview' : 'develop/overview.md'
- 'Setup for Development' : 'develop/Development-Environment.md'
- 'Architecture Overview' : 'develop/overview.md'
- 'OSM Data Import' : 'develop/Import.md'
- 'Place Ranking' : 'develop/Ranking.md'
- 'Postcodes' : 'develop/Postcodes.md'
- 'Setup Test Environment' : 'develop/Setup.md'
- 'Building Documentation' : 'develop/Documentation.md'
- 'Testing' : 'develop/Testing.md'
- 'External Data Sources': 'develop/data-sources.md'
- 'Appendix':
- 'Installation on CentOS 7' : 'appendix/Install-on-Centos-7.md'

2
settings/defaults.php
View File

@@ -85,7 +85,7 @@ if (file_exists(CONST_InstallPath.'/settings/local.php')) require_once(CONST_Ins
// Website settings
@define('CONST_NoAccessControl', true);
@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
@define('CONST_Website_BaseURL', '/');
// Language to assume when none is supplied with the query.
// When set to false, the local language (i.e. the name tag without suffix)
// will be used.

7
test/Makefile
View File

@@ -1,9 +1,14 @@
all: bdd php
no-test-db: bdd-no-test-db php
bdd:
cd bdd && behave -DREMOVE_TEMPLATE=1
bdd-no-test-db:
cd bdd && behave -DREMOVE_TEMPLATE=1 db osm2pgsql
php:
cd php && phpunit ./
.PHONY: bdd php
.PHONY: bdd php no-test-db

121
vagrant/Install-on-Ubuntu-18-nginx.sh
View File

@@ -1,121 +0,0 @@
#!/bin/bash
#
# This is variation of Install-on-Ubuntu.sh showcasing how to use the
# nginx webserver instead of Apache2. We might eventually merge both
# files. Right now expect this file to become outdated/unmaintained
# over time.
#
# This file lacks many comments found in Install-on-Ubuntu.sh, you
# should check that file first to get a basic understanding.
#
# hacks for broken vagrant box
sudo rm -f /var/lib/dpkg/lock
sudo update-locale LANG=en_US.UTF-8
export APT_LISTCHANGES_FRONTEND=none
export DEBIAN_FRONTEND=noninteractive
sudo apt-get update -qq
sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev\
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 \
nginx php-fpm php php-pgsql \
php-intl python3-setuptools python3-dev python3-pip \
python3-psycopg2 python3-tidylib git
export USERNAME=vagrant
export USERHOME=/home/vagrant
chmod a+x $USERHOME
# Setting up PostgreSQL
# ---------------------
#
# Tune the postgresql configuration, see same section in Install-on-Ubuntu.sh
sudo systemctl restart postgresql
sudo -u postgres createuser -s $USERNAME
sudo -u postgres createuser www-data
#
# Setting up the Nginx Webserver
# -------------------------------
#
# You need to configure php-fpm to listen on a Unix socket. Then create Nginx
# configuration to forward localhost:80 requests to that socket.
#
sudo tee /etc/php/7.2/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF
[www]
; Comment out the tcp listener and add the unix socket
;listen = 127.0.0.1:9000
listen = /var/run/php7.2-fpm.sock
; Ensure that the daemon runs as the correct user
listen.owner = www-data
listen.group = www-data
listen.mode = 0666
; Unix user of FPM processes
user = www-data
group = www-data
; Choose process manager type (static, dynamic, ondemand)
pm = ondemand
pm.max_children = 5
EOF_PHP_FPM_CONF
sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
server {
listen 80 default_server;
listen [::]:80 default_server;
root $USERHOME/build/website;
index search.php index.html;
location / {
try_files \$uri \$uri/ @php;
}
location @php {
fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php";
fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php";
fastcgi_param QUERY_STRING \$args;
fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
}
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f \$document_root\$fastcgi_script_name) {
return 404;
}
fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
fastcgi_index search.php;
include fastcgi.conf;
}
}
EOF_NGINX_CONF
sudo sed -i 's:#.*::' /etc/nginx/sites-available/default
#
# Enable the configuration and restart Nginx
#
sudo systemctl stop apache2 # just in case it's installed as well
sudo systemctl restart php7.2-fpm nginx
# From here continue in the 'Installing Nominatim' section in
# Install-on-Ubuntu.sh

177
vagrant/Install-on-Ubuntu-18.sh
View File

@@ -18,20 +18,19 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# Make sure all packages are are up-to-date by running:
#
#DOCS: :::sh
sudo apt-get -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" --force-yes -fuy install grub-pc #DOCS:
sudo apt-get update -qq
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev\
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
apache2 php php-pgsql libapache2-mod-php \
php-intl python3-setuptools python3-dev python3-pip \
python3-psycopg2 python3-tidylib git
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev\
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php php-pgsql php-intl \
python3-psycopg2 git
#
@@ -87,35 +86,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
sudo -u postgres createuser -s $USERNAME
sudo -u postgres createuser www-data
#
# Setting up the Apache Webserver
# -------------------------------
#
# You need to create an alias to the website directory in your apache
# configuration. Add a separate nominatim configuration to your webserver:
#DOCS:```sh
sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
<Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim $USERHOME/build/website #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website
EOFAPACHECONF
#DOCS:```
sudo sed -i 's:#.*::' /etc/apache2/conf-available/nominatim.conf #DOCS:
#
# Then enable the configuration and restart apache
#
sudo a2enconf nominatim
sudo systemctl restart apache2
#
# Installing Nominatim
# ====================
@@ -143,12 +113,53 @@ fi #DOCS:
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
cd $USERHOME #DOCS: :::sh
cd $USERHOME
mkdir build
cd build
cmake $USERHOME/Nominatim
make
# Nominatim is now ready to use. You can continue with
# [importing a database from OSM data](../admin/Import.md). If you want to set up
# a webserver first, continue reading.
#
# Setting up a webserver
# ======================
#
# Option 1: Using Apache
# ----------------------
#
if [ "x$2" == "xinstall-apache" ]; then #DOCS:
#
# Apache has a PHP module that can be used to serve Nominatim. To install them
# run:
sudo apt install -y apache2 libapache2-mod-php
# You need to create an alias to the website directory in your apache
# configuration. Add a separate nominatim configuration to your webserver:
#DOCS:```sh
sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
<Directory "$USERHOME/build/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim $USERHOME/build/website
EOFAPACHECONF
#DOCS:```
#
# Then enable the configuration and restart apache
#
sudo a2enconf nominatim
sudo systemctl restart apache2
# You need to create a minimal configuration file that tells nominatim
# where it is located on the webserver:
@@ -159,6 +170,88 @@ tee settings/local.php << EOF
EOF
#DOCS:```
# The Nominatim API is now available at `http://localhost/nominatim/`.
# Nominatim is now ready to use. Continue with
# [importing a database from OSM data](../admin/Import.md).
fi #DOCS:
#
# Option 2: Using nginx
# ---------------------
#
if [ "x$2" == "xinstall-nginx" ]; then #DOCS:
# Nginx has no native support for php scripts. You need to set up php-fpm for
# this purpose. First install nginx and php-fpm:
sudo apt install -y nginx php-fpm
# You need to configure php-fpm to listen on a Unix socket.
#DOCS:```sh
sudo tee /etc/php/7.2/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF
[www]
; Replace the tcp listener and add the unix socket
listen = /var/run/php7.2-fpm.sock
; Ensure that the daemon runs as the correct user
listen.owner = www-data
listen.group = www-data
listen.mode = 0666
; Unix user of FPM processes
user = www-data
group = www-data
; Choose process manager type (static, dynamic, ondemand)
pm = ondemand
pm.max_children = 5
EOF_PHP_FPM_CONF
#DOCS:```
# Then create a Nginx configuration to forward http requests to that socket.
#DOCS:```sh
sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
server {
listen 80 default_server;
listen [::]:80 default_server;
root $USERHOME/build/website;
index search.php index.html;
location / {
try_files \$uri \$uri/ @php;
}
location @php {
fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php";
fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php";
fastcgi_param QUERY_STRING \$args;
fastcgi_pass unix:/var/run/php7.2-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
}
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f \$document_root\$fastcgi_script_name) {
return 404;
}
fastcgi_pass unix:/var/run/php7.2-fpm.sock;
fastcgi_index search.php;
include fastcgi.conf;
}
}
EOF_NGINX_CONF
#DOCS:```
#
# Enable the configuration and restart Nginx
#
sudo systemctl restart php7.2-fpm nginx
# The Nominatim API is now available at `http://localhost/`.
fi #DOCS:

175
vagrant/Install-on-Ubuntu-20.sh
View File

@@ -22,18 +22,18 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
-o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" \ #DOCS:
--allow-downgrades --allow-remove-essential --allow-change-held-packages \ #DOCS:
-fuy install grub-pc #DOCS:
sudo apt-get update -qq
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-12 postgresql-12-postgis-3 \
postgresql-contrib postgresql-12-postgis-3-scripts \
apache2 php php-pgsql libapache2-mod-php \
php-intl python3-setuptools python3-dev python3-pip \
python3-psycopg2 python3-tidylib git
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-12 postgresql-12-postgis-3 \
postgresql-contrib postgresql-12-postgis-3-scripts \
php php-pgsql php-intl \
python3-psycopg2 git
#
# System Configuration
@@ -88,35 +88,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
sudo -u postgres createuser -s $USERNAME
sudo -u postgres createuser www-data
#
# Setting up the Apache Webserver
# -------------------------------
#
# You need to create an alias to the website directory in your apache
# configuration. Add a separate nominatim configuration to your webserver:
#DOCS:```sh
sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
<Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim $USERHOME/build/website #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website
EOFAPACHECONF
#DOCS:```
sudo sed -i 's:#.*::' /etc/apache2/conf-available/nominatim.conf #DOCS:
#
# Then enable the configuration and restart apache
#
sudo a2enconf nominatim
sudo systemctl restart apache2
#
# Installing Nominatim
# ====================
@@ -144,12 +115,52 @@ fi #DOCS:
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
cd $USERHOME #DOCS: :::sh
cd $USERHOME
mkdir build
cd build
cmake $USERHOME/Nominatim
make
# Nominatim is now ready to use. You can continue with
# [importing a database from OSM data](../admin/Import.md). If you want to set up
# a webserver first, continue reading.
#
# Setting up a webserver
# ======================
#
# Option 1: Using Apache
# ----------------------
#
if [ "x$2" == "xinstall-apache" ]; then #DOCS:
#
# Apache has a PHP module that can be used to serve Nominatim. To install them
# run:
sudo apt install -y apache2 libapache2-mod-php
# You need to create an alias to the website directory in your apache
# configuration. Add a separate nominatim configuration to your webserver:
#DOCS:```sh
sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
<Directory "$USERHOME/build/website">
Options FollowSymLinks MultiViews
AddType text/html .php
DirectoryIndex search.php
Require all granted
</Directory>
Alias /nominatim $USERHOME/build/website
EOFAPACHECONF
#DOCS:```
#
# Then enable the configuration and restart apache
#
sudo a2enconf nominatim
sudo systemctl restart apache2
# You need to create a minimal configuration file that tells nominatim
# where it is located on the webserver:
@@ -160,6 +171,88 @@ tee settings/local.php << EOF
EOF
#DOCS:```
# The Nominatim API is now available at `http://localhost/nominatim/`.
# Nominatim is now ready to use. Continue with
# [importing a database from OSM data](../admin/Import.md).
fi #DOCS:
#
# Option 2: Using nginx
# ---------------------
#
if [ "x$2" == "xinstall-nginx" ]; then #DOCS:
# Nginx has no native support for php scripts. You need to set up php-fpm for
# this purpose. First install nginx and php-fpm:
sudo apt install -y nginx php-fpm
# You need to configure php-fpm to listen on a Unix socket.
#DOCS:```sh
sudo tee /etc/php/7.4/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF
[www]
; Replace the tcp listener and add the unix socket
listen = /var/run/php7.4-fpm.sock
; Ensure that the daemon runs as the correct user
listen.owner = www-data
listen.group = www-data
listen.mode = 0666
; Unix user of FPM processes
user = www-data
group = www-data
; Choose process manager type (static, dynamic, ondemand)
pm = ondemand
pm.max_children = 5
EOF_PHP_FPM_CONF
#DOCS:```
# Then create a Nginx configuration to forward http requests to that socket.
#DOCS:```sh
sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
server {
listen 80 default_server;
listen [::]:80 default_server;
root $USERHOME/build/website;
index search.php index.html;
location / {
try_files \$uri \$uri/ @php;
}
location @php {
fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php";
fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php";
fastcgi_param QUERY_STRING \$args;
fastcgi_pass unix:/var/run/php7.4-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
}
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
if (!-f \$document_root\$fastcgi_script_name) {
return 404;
}
fastcgi_pass unix:/var/run/php7.4-fpm.sock;
fastcgi_index search.php;
include fastcgi.conf;
}
}
EOF_NGINX_CONF
#DOCS:```
#
# Enable the configuration and restart Nginx
#
sudo systemctl restart php7.4-fpm nginx
# The Nominatim API is now available at `http://localhost/`.
fi #DOCS: