From ff6c3a705be5477d32d832df174107e65c3556a5 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 1 May 2016 22:45:54 +0200 Subject: [PATCH 1/9] initial version of documented CentOS7 vagrant script --- CMakeLists.txt | 1 + Vagrantfile | 4 +- docs/CMakeLists.txt | 18 ++++ vagrant/install-on-centos-7.sh | 189 +++++++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 docs/CMakeLists.txt create mode 100755 vagrant/install-on-centos-7.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e11e7b8..1aaf05f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,5 +127,6 @@ add_test(NAME php add_subdirectory(module) add_subdirectory(nominatim) +add_subdirectory(docs) #----------------------------------------------------------------------------- diff --git a/Vagrantfile b/Vagrantfile index f32640f1..889bad36 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ Vagrant.configure("2") do |config| # If true, then any SSH connections made will enable agent forwarding. config.ssh.forward_agent = true - config.vm.synced_folder ".", "/home/vagrant/Nominatim" + #config.vm.synced_folder ".", "/home/vagrant/Nominatim" config.vm.define "ubuntu" do |sub| sub.vm.box = "ubuntu/trusty64" @@ -20,7 +20,7 @@ Vagrant.configure("2") do |config| end config.vm.define "centos" do |sub| sub.vm.box = "bento/centos-7.2" - sub.vm.provision :shell, :path => "vagrant/centos-7-provision.sh" + sub.vm.provision :shell, :path => "vagrant/install-on-centos-7.sh" end # configure shared package cache if possible diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt new file mode 100644 index 00000000..fb121c8f --- /dev/null +++ b/docs/CMakeLists.txt @@ -0,0 +1,18 @@ +# Auto-generated vagrant install documentation + +set (INSTALLDOCFILES + install-on-centos-7 + ) + +foreach (df ${INSTALLDOCFILES}) + ADD_CUSTOM_COMMAND( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${df}.md + COMMAND sed [=['/^#!/d;s:^#\( \|$$\)::;s/.*#DOCS://']=] ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh > ${CMAKE_CURRENT_BINARY_DIR}/${df}.md + MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh + COMMENT "Creating markdown docs from vagrant/${df}.sh" + ) + +ADD_CUSTOM_TARGET( md_install_${df} ALL + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${df}.md + ) +endforeach() + diff --git a/vagrant/install-on-centos-7.sh b/vagrant/install-on-centos-7.sh new file mode 100755 index 00000000..1780a6cd --- /dev/null +++ b/vagrant/install-on-centos-7.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# +# Installing the Required Software +# ================================ +# +# *Note:* these installation instructions are also available in executable +# form for use with vagrant in the vagrant/ directory. +# +# These instructions expect that you have a freshly installed CentOS version 7. +# Make sure all packages are are up-to-date by running: +# + sudo yum update -y +# +# Setting up Repositories +# ----------------------- +# +# The standard CentOS repositories don't contain all the required packages, +# you need to enable the EPEL repository as well. To enable it on CentOS, +# install the epel-release RPM by running: + + sudo yum install -y epel-release + +# +# Getting the Software Packages +# ----------------------------- +# +# Now you can install all packages needed for Nominatim: + + sudo yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \ + git cmake make gcc gcc-c++ libtool policycoreutils-python \ + php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \ + bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel + +# +# System Configuration +# ==================== +# +# The following steps are meant to configure a fresh CentOS installation +# for use with Nominatim. You may skip some of the steps if you have your +# OS already configured. +# +# Creating Dedicated User Accounts +# -------------------------------- +# +# Nominatim will run as a global service on your machine. It is therefore +# best to install it under its own separate user account. In the following +# we assume this user is called nominatim and the installation will be in +# /srv/nominatim. To create the user and directory run: +# +# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim +# +# You may find a more suitable location if you wish. +# +# To be able to copy and paste instructions from this manual, export +# user name and home directory now like this: +# + export USERNAME=vagrant #DOCS: export USERNAME=nominatim + export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim +# +# **Never, ever run the installation as a root user.** You have been warned. +# +# Setting up PostgreSQL +# --------------------- +# +# CentOS does not automatically create a database cluster. Therefore, start +# with initializing the database, then enable the server to start at boot: + + sudo postgresql-setup initdb + sudo systemctl enable postgresql + +# +# Next tune the postgresql configuration, which is located in +# `/var/lib/pgsql/data/postgresql.conf`. See +# [the wiki](http://wiki.openstreetmap.org/wiki/Nominatim/Installation#PostgreSQL_Tuning_and_Configuration) for the parameters to change. +# +# Now start the postgresql service after updating this config file. + + sudo systemctl restart postgresql + +# +# Finally, we need to add two postgres users: one for the user that does +# the import and another for the webserver ro access the database: +# + + sudo -u postgres createuser -s $USERNAME + sudo -u postgres apache + +# +# Setting up the Apache Webserver +# ------------------------------- +# +# You need to create an alias to the website directory in your apache +# configuration. This can be most easily done by XXX + +# +# Adding SELinux Security Settings +# -------------------------------- +# +# It is a good idea to leave SELinux enabled and enforcing, particularly +# with a web server accessible from the Internet. At a minimum the +# following SELinux labeling should be done for Nominatim: + + sudo semanage fcontext -a -t httpd_sys_content_t "$USERHOME/Nominatim/(website|lib|settings)(/.*)?" + sudo semanage fcontext -a -t lib_t "$USERHOME/Nominatim/module/nominatim.so" + sudo restorecon -R -v $USERHOME/Nominatim + +# +# Installing Nominatim +# ==================== +# +# Building and Configuration +# -------------------------- +# +# Get the source code from Github and change into the source directory +# + + cd $USERHOME + sudo -u $USERNAME git clone --recursive git://github.com/twain47/Nominatim.git + +# The code is built in a special directory. Create this directory, +# then configure and build Nominatim in there: + +#DOCS: cd Nominatim + sudo -u $USERNAME mkdir build + cd build + sudo -u $USERNAME cmake ../Nominatim + sudo -u $USERNAME make + +# You need to create a minimal configuration file that tells nominatim +# the name of your webserver user: + +#DOCS:``` +sudo -u $USERNAME tee << EOF + --all [--osm2pgsql-cache 28000] 2>&1 | tee setup.log +# +# The --osm2pgsql-cache parameter is optional but strongly recommended for +# planet imports. It sets the node cache size for the osm2pgsql import part +# (see -C parameter in osm2pgsql help). 28GB are recommended for a full planet +# imports, for excerpts you can use less. +# Adapt to your available RAM to avoid swapping. +# +# The import will take as little as an hour for a small country extract +# and as much as 10 days for a full-scale planet import on less powerful +# hardware. +# +# +# Loading Additional Datasets +# --------------------------- +# +# The following commands will create additional entries for countries and POI searches: +# +# ./utils/specialphrases.php --countries > data/specialphrases_countries.sql +# psql -d nominatim -f data/specialphrases_countries.sql +# ./utils/specialphrases.php --wiki-import > data/specialphrases.sql +# psql -d nominatim -f data/specialphrases.sql + From d1b1acaf1dce32d0e7dfd6879459e51fd6d55535 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 7 Jun 2016 00:17:15 +0200 Subject: [PATCH 2/9] split instruction into software installation and import Basically leaves the part that needs to be done by the automatic vagrant script in the bash script and moves the remaining part into an OS-independent md file. Also fixes some of the instructions for CentOS. Now almost runs through (some minor issue with permission remains). --- Vagrantfile | 6 +- docs/CMakeLists.txt | 11 +++- docs/Import_and_update.md | 77 +++++++++++++++++++++++++ docs/bash2md.sh | 7 +++ vagrant/install-on-centos-7.sh | 101 ++++++++++++--------------------- 5 files changed, 133 insertions(+), 69 deletions(-) create mode 100644 docs/Import_and_update.md create mode 100755 docs/bash2md.sh diff --git a/Vagrantfile b/Vagrantfile index 889bad36..09e7eab9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -3,12 +3,14 @@ Vagrant.configure("2") do |config| # Apache webserver - config.vm.network "forwarded_port", guest: 8089, host: 8089 + config.vm.network "forwarded_port", guest: 80, host: 8089 # If true, then any SSH connections made will enable agent forwarding. config.ssh.forward_agent = true - #config.vm.synced_folder ".", "/home/vagrant/Nominatim" + if ENV['CHECKOUT'] != 'y' then + config.vm.synced_folder ".", "/home/vagrant/Nominatim" + end config.vm.define "ubuntu" do |sub| sub.vm.box = "ubuntu/trusty64" diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index fb121c8f..fd7b808f 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -6,8 +6,9 @@ set (INSTALLDOCFILES foreach (df ${INSTALLDOCFILES}) ADD_CUSTOM_COMMAND( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${df}.md - COMMAND sed [=['/^#!/d;s:^#\( \|$$\)::;s/.*#DOCS://']=] ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh > ${CMAKE_CURRENT_BINARY_DIR}/${df}.md + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh ${CMAKE_CURRENT_BINARY_DIR}/${df}.md MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh COMMENT "Creating markdown docs from vagrant/${df}.sh" ) @@ -16,3 +17,11 @@ ADD_CUSTOM_TARGET( md_install_${df} ALL ) endforeach() +set (GENERALDOCFILES + Import_and_update.md + ) + +foreach (df ${GENERALDOCFILES}) + CONFIGURE_FILE(${df} ${df}) +endforeach() + diff --git a/docs/Import_and_update.md b/docs/Import_and_update.md new file mode 100644 index 00000000..c2dcf937 --- /dev/null +++ b/docs/Import_and_update.md @@ -0,0 +1,77 @@ +Importing a new database +======================== + +The following instructions explain how to create a Nominatim database +from an OSM planet file and how to keep the database up to date. It +is assumed that you have already sucessfully installed the Nominatim +software itself, if not return to the [prerequisites page](Prerequisites.md). + +Configuration setup in settings/local.php +----------------------------------------- + +There are lots of configuration settings you can tweak. Have a look +at `settings/settings.php` for a full list. Most should have a sensible default. +If you plan to import a large dataset (e.g. Europe, North America, planet), +you should also enable flatnode storage of node locations. With this +setting enabled, node coordinates are stored in a simple file instead +of the database. This will save you import time and disk storage. +Add to your settings/local.php: + + @define('CONST_Osm2pgsql_Flatnode_File', '/path/to/flatnode.file'); + + +Downloading additional data +--------------------------- + +### Wikipedia rankings + +Wikipedia can be used as an optional auxiliary data source to help indicate +the importance of osm features. Nominatim will work without this information +but it will improve the quality of the results if this is installed. +This data is available as a binary download: + + cd $NOMINATIM_SOURCE_DIR/data + wget http://www.nominatim.org/data/wikipedia_article.sql.bin + wget http://www.nominatim.org/data/wikipedia_redirect.sql.bin + +Combined the 2 files are around 1.5GB and add around 30GB to the install +size of nominatim. They also increase the install time by an hour or so. + +### UK postcodes + +Nominatim can use postcodes from an external source to improve searches that involve a UK postcode. This data can be optionally downloaded: + + cd $NOMINATIM_SOURCE_DIR/data + wget http://www.nominatim.org/data/gb_postcode_data.sql.gz + + +Initial Import of the Data +-------------------------- + +**Important:** first try the import with a small excerpt, for example from Geofabrik. + +Download the data to import and load the data with the following command: + + ./utils/setup.php --osm-file --all [--osm2pgsql-cache 28000] 2>&1 | tee setup.log + +The --osm2pgsql-cache parameter is optional but strongly recommended for +planet imports. It sets the node cache size for the osm2pgsql import part +(see -C parameter in osm2pgsql help). 28GB are recommended for a full planet +imports, for excerpts you can use less. +Adapt to your available RAM to avoid swapping. + +The import will take as little as an hour for a small country extract +and as much as 10 days for a full-scale planet import on less powerful +hardware. + + +Loading Additional Datasets +--------------------------- + +The following commands will create additional entries for countries and POI searches: + + ./utils/specialphrases.php --countries > data/specialphrases_countries.sql + psql -d nominatim -f data/specialphrases_countries.sql + ./utils/specialphrases.php --wiki-import > data/specialphrases.sql + psql -d nominatim -f data/specialphrases.sql + diff --git a/docs/bash2md.sh b/docs/bash2md.sh new file mode 100755 index 00000000..a62fbd1e --- /dev/null +++ b/docs/bash2md.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# +# Extract markdown-formatted documentation from a source file +# +# Usage: bash2md.sh + +sed '/^#!/d;s:^#\( \|$\)::;s/.*#DOCS://' $1 > $2 diff --git a/vagrant/install-on-centos-7.sh b/vagrant/install-on-centos-7.sh index 1780a6cd..32a12e9e 100755 --- a/vagrant/install-on-centos-7.sh +++ b/vagrant/install-on-centos-7.sh @@ -1,29 +1,22 @@ #!/bin/bash # -# Installing the Required Software -# ================================ -# # *Note:* these installation instructions are also available in executable # form for use with vagrant in the vagrant/ directory. # +# Installing the Required Software +# ================================ +# # These instructions expect that you have a freshly installed CentOS version 7. # Make sure all packages are are up-to-date by running: # sudo yum update -y -# -# Setting up Repositories -# ----------------------- -# + # The standard CentOS repositories don't contain all the required packages, # you need to enable the EPEL repository as well. To enable it on CentOS, # install the epel-release RPM by running: sudo yum install -y epel-release -# -# Getting the Software Packages -# ----------------------------- -# # Now you can install all packages needed for Nominatim: sudo yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \ @@ -83,14 +76,33 @@ # sudo -u postgres createuser -s $USERNAME - sudo -u postgres apache + sudo -u postgres createuser apache # # Setting up the Apache Webserver # ------------------------------- # # You need to create an alias to the website directory in your apache -# configuration. This can be most easily done by XXX +# configuration. Add a separate nominatim configuration to your webserver: + +#DOCS:``` +sudo cat > /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF + #DOCS: + Options FollowSymLinks MultiViews + AddType text/html .php + + +Alias /nominatim $USERHOME/build/website #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website +EOFAPACHECONF +#DOCS:``` + +sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS: + +# +# Then reload apache +# + + sudo systemctl restart httpd # # Adding SELinux Security Settings @@ -113,77 +125,34 @@ # # Get the source code from Github and change into the source directory # +if [ "x$CHECKOUT" == "xy" ]; then #DOCS: cd $USERHOME sudo -u $USERNAME git clone --recursive git://github.com/twain47/Nominatim.git +#DOCS: cd Nominatim + +else #DOCS: + cd $USERHOME #DOCS: +fi #DOCS: # The code is built in a special directory. Create this directory, # then configure and build Nominatim in there: -#DOCS: cd Nominatim sudo -u $USERNAME mkdir build cd build - sudo -u $USERNAME cmake ../Nominatim + sudo -u $USERNAME cmake $USERHOME/Nominatim sudo -u $USERNAME make # You need to create a minimal configuration file that tells nominatim # the name of your webserver user: #DOCS:``` -sudo -u $USERNAME tee << EOF +sudo -u $USERNAME cat > settings/local.php << EOF --all [--osm2pgsql-cache 28000] 2>&1 | tee setup.log -# -# The --osm2pgsql-cache parameter is optional but strongly recommended for -# planet imports. It sets the node cache size for the osm2pgsql import part -# (see -C parameter in osm2pgsql help). 28GB are recommended for a full planet -# imports, for excerpts you can use less. -# Adapt to your available RAM to avoid swapping. -# -# The import will take as little as an hour for a small country extract -# and as much as 10 days for a full-scale planet import on less powerful -# hardware. -# -# -# Loading Additional Datasets -# --------------------------- -# -# The following commands will create additional entries for countries and POI searches: -# -# ./utils/specialphrases.php --countries > data/specialphrases_countries.sql -# psql -d nominatim -f data/specialphrases_countries.sql -# ./utils/specialphrases.php --wiki-import > data/specialphrases.sql -# psql -d nominatim -f data/specialphrases.sql +Nominatim is now ready to use. Continue with +[importing a database from OSM data](Import_and_update.md). From 186a633185c07622640792c8da0591d6c2f5a5f8 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 7 Jun 2016 00:27:20 +0200 Subject: [PATCH 3/9] fix apache permission problems in CentOS vagrant script --- vagrant/install-on-centos-7.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vagrant/install-on-centos-7.sh b/vagrant/install-on-centos-7.sh index 32a12e9e..4f8f917e 100755 --- a/vagrant/install-on-centos-7.sh +++ b/vagrant/install-on-centos-7.sh @@ -52,6 +52,10 @@ # # **Never, ever run the installation as a root user.** You have been warned. # +# Make sure that system servers can read from the home directory: + + chmod a+x $USERHOME + # Setting up PostgreSQL # --------------------- # @@ -90,6 +94,7 @@ sudo cat > /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF #DOCS: Options FollowSymLinks MultiViews AddType text/html .php + Require all granted Alias /nominatim $USERHOME/build/website #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website From f71e9dd187f697e31dddacb1cdaea0d6d29009e9 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 7 Jun 2016 22:47:57 +0200 Subject: [PATCH 4/9] move installation instructions from wiki page here --- README => README.md | 26 ++++---- docs/CMakeLists.txt | 1 + docs/Import_and_update.md | 112 ++++++++++++++++++++++++++++++--- vagrant/install-on-centos-7.sh | 6 +- 4 files changed, 123 insertions(+), 22 deletions(-) rename README => README.md (64%) diff --git a/README b/README.md similarity index 64% rename from README rename to README.md index 7a760ff6..c5a01a36 100644 --- a/README +++ b/README.md @@ -12,22 +12,15 @@ Documentation ============= More information about Nominatim, including usage and installation instructions, -can be found in the OSM wiki at: +can be found in the docs/ subdirectory and in the OSM wiki at: http://wiki.openstreetmap.org/wiki/Nominatim Installation ============ -The following instructions is a quick guide to installation. A more detailed guide -how to set up your own instance of Nominatim can be found in the wiki: - -http://wiki.openstreetmap.org/wiki/Nominatim/Installation - -Note that this repository contains a submodule called osm2pgsql. Make sure it -is cloned as well by running `git submodule update --init`. - -Installation steps: +There are detailed installation instructions in the /docs directory. +Here is a quick summary of the necessary steps. 1. Compile Nominatim: @@ -36,11 +29,20 @@ Installation steps: cmake .. make + For more detailed installation instructions see [docs/installation.md](). + There are also step-by-step instructions for + [Ubuntu 16.04](docs/install-on-ubuntu-16.md) and + [CentOS 7.2](docs/install-on-centos-7.md). + 2. Get OSM data and import: - ./utils/setup.php --osm-file --all + ./build/utils/setup.php --osm-file --all + + Details can be found in [docs/Import_and_update.md]() + +3. Point your webserver to the ./build/website directory. + -3. Point your webserver to the ./website directory. License ======= diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index fd7b808f..f83f10d7 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -18,6 +18,7 @@ ADD_CUSTOM_TARGET( md_install_${df} ALL endforeach() set (GENERALDOCFILES + Installation.md Import_and_update.md ) diff --git a/docs/Import_and_update.md b/docs/Import_and_update.md index c2dcf937..80e8bf47 100644 --- a/docs/Import_and_update.md +++ b/docs/Import_and_update.md @@ -4,7 +4,7 @@ Importing a new database The following instructions explain how to create a Nominatim database from an OSM planet file and how to keep the database up to date. It is assumed that you have already sucessfully installed the Nominatim -software itself, if not return to the [prerequisites page](Prerequisites.md). +software itself, if not return to the [installation page](Installation.md). Configuration setup in settings/local.php ----------------------------------------- @@ -57,12 +57,8 @@ Download the data to import and load the data with the following command: The --osm2pgsql-cache parameter is optional but strongly recommended for planet imports. It sets the node cache size for the osm2pgsql import part (see -C parameter in osm2pgsql help). 28GB are recommended for a full planet -imports, for excerpts you can use less. -Adapt to your available RAM to avoid swapping. - -The import will take as little as an hour for a small country extract -and as much as 10 days for a full-scale planet import on less powerful -hardware. +imports, for excerpts you can use less. Adapt to your available RAM to +avoid swapping, never give more than 2/3 of RAM to osm2pgsql. Loading Additional Datasets @@ -75,3 +71,105 @@ The following commands will create additional entries for countries and POI sear ./utils/specialphrases.php --wiki-import > data/specialphrases.sql psql -d nominatim -f data/specialphrases.sql + +Installing Tiger housenumber data for the US +============================================ + +Nominatim is able to use the official TIGER address set to complement the +OSM housenumber data in the US. You can add TIGER data to your own Nominatim +instance by following these steps: + + 1. Install the GDAL library and python bindings + + Ubuntu: apt-get install python-gdal + CentOS: yum install gdal-python + + 2. Get the TIGER 2015 data. You will need the EDGES files + (3,234 zip files, 11GB total). + + wget -r ftp://mirror1.shellbot.com/census/geo/tiger/TIGER2015/EDGES/ + (1gb/s preferred mirror) MIRROR HOSTED BY SHELLBOT, LTD. + + OR: + + wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2015/EDGES/ + (500kb/s original source) HOSTED BY THE US CENSUS BUREAU + + 3. Convert the data into SQL statements (stored in data/tiger): + + ./utils/imports.php --parse-tiger + + 4. Import the data into your Nominatim database: + + ./utils/setup.php --import-tiger-data + +Be warned that the import can take a very long time, especially if you +import all of the US. + + +Updates +======= + +There are many different possibilities to update your Nominatim database. +The following section describes how to keep it up-to-date with osmosis. +For a list of other methods see the output of ./utils/update.php --help. + +Installing the newest version of osmosis +---------------------------------------- + +The version of osmosis that comes with your distribution should be sufficient +in most cases. + +If you want to install it by hand, get the latest version from the +[Osmosis website](http://wiki.openstreetmap.org/wiki/Osmosis). Then +tell Nominatim to use this version by adding the following line to +your `settings/local.php`: + + @define('CONST_Osmosis_Binary', '/usr/local/bin/osmosis'); + +Setting up the update process +----------------------------- + +Next the update needs to be initialised. By default Nominatim is configured +to update using the global minutely diffs. + +If you want a different update source you will need to add some settings +to `settings/local.php`. For example, to use the daily country extracts +diffs for Ireland from geofabrik add the following: + + // base URL of the replication service + @define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/ireland-and-northern-ireland-updates'); + // Process each update separately, osmosis cannot merge multiple updates + @define('CONST_Replication_MaxInterval', '40000'); + // How often upstream publishes diffs + @define('CONST_Replication_Update_Interval', '86400'); + // How long to sleep if no update found yet + @define('CONST_Replication_Recheck_Interval', '900'); + + +Delete existing 'settings/configuration.txt' then run the following command +to create the osmosis configuration files: + + ./utils/setup.php --osmosis-init + +Enabling hierarchical updates +----------------------------- + +When a place is updated in the database, all places that contain this place +in their address need to be updated as well. These hierarchical updates are +disabled by default because they slow down the initial import. +Enable them with the following command: + + ./utils/setup.php --create-functions --enable-diff-updates + +Updating Nominatim +------------------ + +The following command will keep your database constantly up to date: + + ./utils/update.php --import-osmosis-all --no-npi + +If you have imported multiple country extracts and want to keep them +up-to-date, have a look at the script in +(issue #60)[https://github.com/twain47/Nominatim/issues/60]. + diff --git a/vagrant/install-on-centos-7.sh b/vagrant/install-on-centos-7.sh index 4f8f917e..afdeccd9 100755 --- a/vagrant/install-on-centos-7.sh +++ b/vagrant/install-on-centos-7.sh @@ -1,7 +1,7 @@ #!/bin/bash # # *Note:* these installation instructions are also available in executable -# form for use with vagrant in the vagrant/ directory. +# form for use with vagrant under vagrant/install-on-centos-7.sh. # # Installing the Required Software # ================================ @@ -67,8 +67,8 @@ # # Next tune the postgresql configuration, which is located in -# `/var/lib/pgsql/data/postgresql.conf`. See -# [the wiki](http://wiki.openstreetmap.org/wiki/Nominatim/Installation#PostgreSQL_Tuning_and_Configuration) for the parameters to change. +# `/var/lib/pgsql/data/postgresql.conf`. See section *Postgres Tuning* in +# [the installation page](Installation.md) for the parameters to change. # # Now start the postgresql service after updating this config file. From 0adb8cb765efc7107b6f500764732f85809a97bf Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 8 Jun 2016 22:51:56 +0200 Subject: [PATCH 5/9] fix cut&paste error in php tests --- tests-php/Nominatim/NominatimTest.php | 61 --------------------------- 1 file changed, 61 deletions(-) diff --git a/tests-php/Nominatim/NominatimTest.php b/tests-php/Nominatim/NominatimTest.php index 8213be00..00fdc132 100644 --- a/tests-php/Nominatim/NominatimTest.php +++ b/tests-php/Nominatim/NominatimTest.php @@ -189,69 +189,8 @@ class NominatimTest extends \PHPUnit_Framework_TestCase ); } - } - - - public function test_geometryText2Points() - { - $fRadius = 1; - - // invalid value - $this->assertEquals( - NULL, - geometryText2Points('', $fRadius) - ); - - - // POINT - $aPoints = geometryText2Points('POINT(10 20)', $fRadius); - $this->assertEquals( - 101, - count($aPoints) - ); - - $this->assertEquals( - - array( - ['', 10, 21], - ['', 10.062790519529, 20.998026728428], - ['', 10.125333233564, 20.992114701314] - ), - array_splice($aPoints, 0,3) - ); - - // POLYGON - $this->assertEquals( - array( - ['30 10', '30', '10'], - ['40 40', '40', '40'], - ['20 40', '20', '40'], - ['10 20', '10', '20'], - ['30 10', '30', '10'] - ), - geometryText2Points('POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))', $fRadius) - ); - - // MULTIPOLYGON - // only the first polygon is used - $this->assertEquals( - array( - ['30 20', '30', '20'], - ['45 40', '45', '40'], - ['10 40', '10', '40'], - ['30 20', '30', '20'], - - // ['15 5' , '15', '5' ], - // ['45 10', '45', '10'], - // ['10 20', '10', '20'], - // ['5 10' , '5' , '10'], - // ['15 5' , '15', '5' ] - ), - geometryText2Points('MULTIPOLYGON(((30 20, 45 40, 10 40, 30 20)),((15 5, 40 10, 10 20, 5 10, 15 5)))', $fRadius) - ); - // you might say we're creating a circle public function test_createPointsAroundCenter() { From 2e08a615655724063ed3e31c766ae666a9df2c57 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 8 Jun 2016 23:03:11 +0200 Subject: [PATCH 6/9] add vagrant script for ubuntu 16 and polish everything --- Vagrantfile | 21 +++-- docs/Installation.md | 151 ++++++++++++++++++++++++++++++ vagrant/install-on-centos-7.sh | 29 ++++-- vagrant/install-on-ubuntu-16.sh | 157 ++++++++++++++++++++++++++++++++ 4 files changed, 341 insertions(+), 17 deletions(-) create mode 100644 docs/Installation.md create mode 100755 vagrant/install-on-ubuntu-16.sh diff --git a/Vagrantfile b/Vagrantfile index 09e7eab9..96d03a2e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,21 +8,28 @@ Vagrant.configure("2") do |config| # If true, then any SSH connections made will enable agent forwarding. config.ssh.forward_agent = true + checkout = "yes" if ENV['CHECKOUT'] != 'y' then config.vm.synced_folder ".", "/home/vagrant/Nominatim" + checkout = "no" end config.vm.define "ubuntu" do |sub| - sub.vm.box = "ubuntu/trusty64" - sub.vm.provision :shell, :path => "vagrant/ubuntu-trusty-provision.sh" - end - config.vm.define "ubuntu-php7" do |sub| - sub.vm.box = "ubuntu/trusty64" - sub.vm.provision :shell, :path => "vagrant/ubuntu-trusty-php7-provision.sh" + sub.vm.box = "bento/ubuntu-16.04" + sub.vm.provision :shell do |s| + s.path = "vagrant/install-on-ubuntu-16.sh" + s.privileged = false + s.args = [checkout] + end end + config.vm.define "centos" do |sub| sub.vm.box = "bento/centos-7.2" - sub.vm.provision :shell, :path => "vagrant/install-on-centos-7.sh" + sub.vm.provision :shell do |s| + s.path = "vagrant/install-on-centos-7.sh" + s.privileged = false + s.args = [checkout] + end end # configure shared package cache if possible diff --git a/docs/Installation.md b/docs/Installation.md new file mode 100644 index 00000000..ddd0fc7b --- /dev/null +++ b/docs/Installation.md @@ -0,0 +1,151 @@ +Nominatim Installation +====================== + +This page contains generic installation instructions for Nominatim and its +prerequisites. There are also step-by-step instructions available for +the following operating systems: + + * [Ubuntu 16.04](install-on-ubuntu-16.md) + * [CentOS 7.2](install-on-centos-7.md) + +These OS-specific instructions can also be found in executable form +in the `vagrant/` directory. + +Prerequisites +------------- + +### Software + +For compiling: + + * [cmake](https://cmake.org/) + * [libxml2](http://xmlsoft.org/) + * a recent C++ compiler + +Nominatim comes with its own version of osm2pgsql. See the +[osm2pgsql README](../osm2pgsql/README.md) for additional dependencies +required for compiling osm2pgsql. + +For running tests: + + * [lettuce](http://lettuce.it) + * [Shapely](http://toblerity.org/shapely/index.html) + * [Psycopg2](http://initd.org/psycopg) + * [nose](https://nose.readthedocs.io) + * [phpunit](https://phpunit.de) + +For running Nominatim: + + * [PostgreSQL](http://www.postgresql.org) (9.1 or later) + * [PostGIS](http://postgis.refractions.net) (2.0 or later) + * [PHP](http://php.net) + * PHP-pgsql + * [PEAR::DB](http://pear.php.net/package/DB) + * a webserver (apache or nginx are recommended) + +For running continuous updates: + + * [osmosis](http://wiki.openstreetmap.org/wiki/Osmosis) + +### Hardware + +A minimum of 2GB of RAM is required or installation will fail. For a full +planet import 32GB of RAM or more strongly are recommended. + +For a full planet install you will need about 500GB of hard disk space (as of +June 2016, take into account that the OSM database is growing fast). SSD disks +will help considerably to speed up import and queries. + +On a 6-core machine with 32GB RAM and SSDs the import of a full planet takes +a bit more than 2 days. Without SSDs 7-8 days are more realistic. + + +Setup of the Server +------------------- + +### PostgreSQL Tuning + +You might want to tune your PostgreSQL installation so that the later steps +make best use of your hardware. You should tune the following parameters in +your `postgresql.conf` file. + + shared_buffers (2GB) + maintenance_work_mem (10GB) + work_mem (50MB) + effective_cache_size (24GB) + synchronous_commit = off + checkpoint_segments = 100 + checkpoint_timeout = 10min + checkpoint_completion_target = 0.9 + +The numbers in brackets behind some parameters seem to work fine for +32GB RAM machine. Adjust to your setup. + +For the initial import, you should also set: + + fsync = off + full_page_writes = off + +Don't forget to reenable them after the initial import or you risk database +corruption. Autovacuum must not be switched off because it ensures that the +tables are frequently analysed. + +### Webserver setup + +The `website/` directory in the build directory contains the configured +website. Include the directory into your webbrowser to serve php files +from there. + +#### Configure for use with Apache + +Make sure your Apache configuration contains the required permissions for the +directory and create an alias: + + + Options FollowSymLinks MultiViews + AddTpe text/html .php + Require all granted + + Alias /nominatim /srv/nominatim/build/website + +`/srv/nominatim/build` should be replaced with the location of your +build directory. + +After making changes in the apache config you need to restart apache. +The website should now be available on http://localhost/nominatim. + +#### Configure for use with Nginx + +Use php-fpm as a deamon for serving PHP cgi. Install php-fpm together with nginx. + +By default php listens on a network socket. If you want it to listen to a +Unix socket instead, change the pool configuration (`pool.d/www.conf`) as +follows: + + ; Comment out the tcp listener and add the unix socket + ;listen = 127.0.0.1:9000 + listen = /var/run/php5-fpm.sock + + ; Ensure that the daemon runs as the correct user + listen.owner = www-data + listen.group = www-data + listen.mode = 0666 + +Tell nginx that php files are special and to fastcgi_pass to the php-fpm +unix socket by adding the location definition to the default configuration. + + location ~ [^/]\.php(/|$) { + fastcgi_split_path_info ^(.+?\.php)(/.*)$; + if (!-f $document_root$fastcgi_script_name) { + return 404; + } + fastcgi_pass unix:/var/run/php5-fpm.sock; + fastcgi_index index.php; + include fastcgi.conf; + } + +Restart the nginx and php5-fpm services and the website should now be available +on http://localhost/. + + +Now continue with [importing the database](Import_and_update.md). diff --git a/vagrant/install-on-centos-7.sh b/vagrant/install-on-centos-7.sh index afdeccd9..d76a816f 100755 --- a/vagrant/install-on-centos-7.sh +++ b/vagrant/install-on-centos-7.sh @@ -24,6 +24,14 @@ php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \ bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel +# If you want to run the test suite, you need to install the following +# aditional packages: + + sudo yum install -y python-pip python-Levenshtein python-psycopg2 \ + php-phpunit-PHPUnit + pip install --user --upgrade pip setuptools lettuce==0.2.18 six==1.9 \ + haversine Shapely pytidylib + # # System Configuration # ==================== @@ -90,7 +98,7 @@ # configuration. Add a separate nominatim configuration to your webserver: #DOCS:``` -sudo cat > /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF +sudo tee /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF #DOCS: Options FollowSymLinks MultiViews AddType text/html .php @@ -130,10 +138,10 @@ sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS: # # Get the source code from Github and change into the source directory # -if [ "x$CHECKOUT" == "xy" ]; then #DOCS: +if [ "x$1" == "xyes" ]; then #DOCS: cd $USERHOME - sudo -u $USERNAME git clone --recursive git://github.com/twain47/Nominatim.git + git clone --recursive git://github.com/twain47/Nominatim.git #DOCS: cd Nominatim else #DOCS: @@ -143,21 +151,22 @@ fi #DOCS: # The code is built in a special directory. Create this directory, # then configure and build Nominatim in there: - sudo -u $USERNAME mkdir build + mkdir build cd build - sudo -u $USERNAME cmake $USERHOME/Nominatim - sudo -u $USERNAME make + cmake $USERHOME/Nominatim + make # You need to create a minimal configuration file that tells nominatim -# the name of your webserver user: +# the name of your webserver user and the URL of the website: #DOCS:``` -sudo -u $USERNAME cat > settings/local.php << EOF +tee settings/local.php << EOF #DOCS: + Options FollowSymLinks MultiViews + AddType text/html .php + Require all granted + + +Alias /nominatim $USERHOME/build/website #DOCS:Alias /nominatim $USERHOME/Nominatim/build/website +EOFAPACHECONF +#DOCS:``` + +sudo sed -i 's:#.*::' /etc/apache2/conf-available/nominatim.conf #DOCS: + +# +# Then enable the configuration and restart apache +# + + sudo a2enconf nominatim + sudo systemctl restart apache2 + +# +# Installing Nominatim +# ==================== +# +# Building and Configuration +# -------------------------- +# +# Get the source code from Github and change into the source directory +# +if [ "x$1" == "xyes" ]; then #DOCS: + + cd $USERHOME + git clone --recursive git://github.com/twain47/Nominatim.git +#DOCS: cd Nominatim + +else #DOCS: + cd $USERHOME #DOCS: +fi #DOCS: + +# The code is built in a special directory. Create this directory, +# then configure and build Nominatim in there: + + mkdir build + cd build + cmake $USERHOME/Nominatim + make + +# You need to create a minimal configuration file that tells nominatim +# where it is located on the webserver: + +#DOCS:``` +tee settings/local.php << EOF + Date: Wed, 8 Jun 2016 23:06:15 +0200 Subject: [PATCH 7/9] add compiled vagrant docs in docs/ directory --- docs/CMakeLists.txt | 1 + docs/install-on-centos-7.md | 171 +++++++++++++++++++++++++++++++++++ docs/install-on-ubuntu-16.md | 156 ++++++++++++++++++++++++++++++++ 3 files changed, 328 insertions(+) create mode 100644 docs/install-on-centos-7.md create mode 100644 docs/install-on-ubuntu-16.md diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index f83f10d7..bf453a77 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -2,6 +2,7 @@ set (INSTALLDOCFILES install-on-centos-7 + install-on-ubuntu-16 ) foreach (df ${INSTALLDOCFILES}) diff --git a/docs/install-on-centos-7.md b/docs/install-on-centos-7.md new file mode 100644 index 00000000..7c3bc97b --- /dev/null +++ b/docs/install-on-centos-7.md @@ -0,0 +1,171 @@ + +*Note:* these installation instructions are also available in executable + form for use with vagrant under vagrant/install-on-centos-7.sh. + +Installing the Required Software +================================ + +These instructions expect that you have a freshly installed CentOS version 7. +Make sure all packages are are up-to-date by running: + + sudo yum update -y + +The standard CentOS repositories don't contain all the required packages, +you need to enable the EPEL repository as well. To enable it on CentOS, +install the epel-release RPM by running: + + sudo yum install -y epel-release + +Now you can install all packages needed for Nominatim: + + sudo yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \ + git cmake make gcc gcc-c++ libtool policycoreutils-python \ + php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \ + bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel + +If you want to run the test suite, you need to install the following +aditional packages: + + sudo yum install -y python-pip python-Levenshtein python-psycopg2 \ + php-phpunit-PHPUnit + pip install --user --upgrade pip setuptools lettuce==0.2.18 six==1.9 \ + haversine Shapely pytidylib + + +System Configuration +==================== + +The following steps are meant to configure a fresh CentOS installation +for use with Nominatim. You may skip some of the steps if you have your +OS already configured. + +Creating Dedicated User Accounts +-------------------------------- + +Nominatim will run as a global service on your machine. It is therefore +best to install it under its own separate user account. In the following +we assume this user is called nominatim and the installation will be in +/srv/nominatim. To create the user and directory run: + + sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim + +You may find a more suitable location if you wish. + +To be able to copy and paste instructions from this manual, export +user name and home directory now like this: + + export USERNAME=nominatim + export USERHOME=/srv/nominatim + +**Never, ever run the installation as a root user.** You have been warned. + +Make sure that system servers can read from the home directory: + + chmod a+x $USERHOME + +Setting up PostgreSQL +--------------------- + +CentOS does not automatically create a database cluster. Therefore, start +with initializing the database, then enable the server to start at boot: + + sudo postgresql-setup initdb + sudo systemctl enable postgresql + + +Next tune the postgresql configuration, which is located in +`/var/lib/pgsql/data/postgresql.conf`. See section *Postgres Tuning* in +[the installation page](Installation.md) for the parameters to change. + +Now start the postgresql service after updating this config file. + + sudo systemctl restart postgresql + + +Finally, we need to add two postgres users: one for the user that does +the import and another for the webserver ro access the database: + + + sudo -u postgres createuser -s $USERNAME + sudo -u postgres createuser apache + + +Setting up the Apache Webserver +------------------------------- + +You need to create an alias to the website directory in your apache +configuration. Add a separate nominatim configuration to your webserver: + +``` +sudo tee /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF + + Options FollowSymLinks MultiViews + AddType text/html .php + Require all granted + + +Alias /nominatim $USERHOME/Nominatim/build/website +EOFAPACHECONF +``` + + + + +Then reload apache + + + sudo systemctl restart httpd + + +Adding SELinux Security Settings +-------------------------------- + +It is a good idea to leave SELinux enabled and enforcing, particularly +with a web server accessible from the Internet. At a minimum the +following SELinux labeling should be done for Nominatim: + + sudo semanage fcontext -a -t httpd_sys_content_t "$USERHOME/Nominatim/(website|lib|settings)(/.*)?" + sudo semanage fcontext -a -t lib_t "$USERHOME/Nominatim/module/nominatim.so" + sudo restorecon -R -v $USERHOME/Nominatim + + +Installing Nominatim +==================== + +Building and Configuration +-------------------------- + +Get the source code from Github and change into the source directory + + + + cd $USERHOME + git clone --recursive git://github.com/twain47/Nominatim.git + cd Nominatim + + + + + +The code is built in a special directory. Create this directory, +then configure and build Nominatim in there: + + mkdir build + cd build + cmake $USERHOME/Nominatim + make + +You need to create a minimal configuration file that tells nominatim +the name of your webserver user and the URL of the website: + +``` +tee settings/local.php << EOF + + Options FollowSymLinks MultiViews + AddType text/html .php + Require all granted + + +Alias /nominatim $USERHOME/Nominatim/build/website +EOFAPACHECONF +``` + + + + +Then enable the configuration and restart apache + + + sudo a2enconf nominatim + sudo systemctl restart apache2 + + +Installing Nominatim +==================== + +Building and Configuration +-------------------------- + +Get the source code from Github and change into the source directory + + + + cd $USERHOME + git clone --recursive git://github.com/twain47/Nominatim.git + cd Nominatim + + + + + +The code is built in a special directory. Create this directory, +then configure and build Nominatim in there: + + mkdir build + cd build + cmake $USERHOME/Nominatim + make + +You need to create a minimal configuration file that tells nominatim +where it is located on the webserver: + +``` +tee settings/local.php << EOF + Date: Wed, 8 Jun 2016 23:17:48 +0200 Subject: [PATCH 8/9] fix formatting of docs --- docs/Import_and_update.md | 41 +++++++++++++++++++-------------------- docs/Installation.md | 6 +++--- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/docs/Import_and_update.md b/docs/Import_and_update.md index 80e8bf47..2f40312a 100644 --- a/docs/Import_and_update.md +++ b/docs/Import_and_update.md @@ -45,10 +45,11 @@ Nominatim can use postcodes from an external source to improve searches that inv wget http://www.nominatim.org/data/gb_postcode_data.sql.gz -Initial Import of the Data +Initial import of the data -------------------------- -**Important:** first try the import with a small excerpt, for example from Geofabrik. +**Important:** first try the import with a small excerpt, for example from +[Geofabrik](http://download.geofabrik.de). Download the data to import and load the data with the following command: @@ -57,19 +58,19 @@ Download the data to import and load the data with the following command: The --osm2pgsql-cache parameter is optional but strongly recommended for planet imports. It sets the node cache size for the osm2pgsql import part (see -C parameter in osm2pgsql help). 28GB are recommended for a full planet -imports, for excerpts you can use less. Adapt to your available RAM to +import, for excerpts you can use less. Adapt to your available RAM to avoid swapping, never give more than 2/3 of RAM to osm2pgsql. -Loading Additional Datasets +Loading additional datasets --------------------------- The following commands will create additional entries for countries and POI searches: - ./utils/specialphrases.php --countries > data/specialphrases_countries.sql - psql -d nominatim -f data/specialphrases_countries.sql - ./utils/specialphrases.php --wiki-import > data/specialphrases.sql - psql -d nominatim -f data/specialphrases.sql + ./utils/specialphrases.php --countries > specialphrases_countries.sql + psql -d nominatim -f specialphrases_countries.sql + ./utils/specialphrases.php --wiki-import > specialphrases.sql + psql -d nominatim -f specialphrases.sql Installing Tiger housenumber data for the US @@ -81,27 +82,25 @@ instance by following these steps: 1. Install the GDAL library and python bindings - Ubuntu: apt-get install python-gdal - CentOS: yum install gdal-python + * Ubuntu: `sudo apt-get install python-gdal` + * CentOS: `sudo yum install gdal-python` 2. Get the TIGER 2015 data. You will need the EDGES files - (3,234 zip files, 11GB total). + (3,234 zip files, 11GB total). Choose one of the two sources: - wget -r ftp://mirror1.shellbot.com/census/geo/tiger/TIGER2015/EDGES/ - (1gb/s preferred mirror) MIRROR HOSTED BY SHELLBOT, LTD. + wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2015/EDGES/ + wget -r ftp://mirror1.shellbot.com/census/geo/tiger/TIGER2015/EDGES/ - OR: - - wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2015/EDGES/ - (500kb/s original source) HOSTED BY THE US CENSUS BUREAU + The first one is the original source, the second a considerably faster + mirror. 3. Convert the data into SQL statements (stored in data/tiger): - ./utils/imports.php --parse-tiger + ./utils/imports.php --parse-tiger 4. Import the data into your Nominatim database: - ./utils/setup.php --import-tiger-data + ./utils/setup.php --import-tiger-data Be warned that the import can take a very long time, especially if you import all of the US. @@ -112,7 +111,7 @@ Updates There are many different possibilities to update your Nominatim database. The following section describes how to keep it up-to-date with osmosis. -For a list of other methods see the output of ./utils/update.php --help. +For a list of other methods see the output of `./utils/update.php --help`. Installing the newest version of osmosis ---------------------------------------- @@ -147,7 +146,7 @@ diffs for Ireland from geofabrik add the following: @define('CONST_Replication_Recheck_Interval', '900'); -Delete existing 'settings/configuration.txt' then run the following command +Delete any existing `settings/configuration.txt`, then run the following command to create the osmosis configuration files: ./utils/setup.php --osmosis-init diff --git a/docs/Installation.md b/docs/Installation.md index ddd0fc7b..fd4f8d0b 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -1,4 +1,4 @@ -Nominatim Installation +Nominatim installation ====================== This page contains generic installation instructions for Nominatim and its @@ -60,10 +60,10 @@ On a 6-core machine with 32GB RAM and SSDs the import of a full planet takes a bit more than 2 days. Without SSDs 7-8 days are more realistic. -Setup of the Server +Setup of the server ------------------- -### PostgreSQL Tuning +### PostgreSQL tuning You might want to tune your PostgreSQL installation so that the later steps make best use of your hardware. You should tune the following parameters in From 6436bab1bea84d851c823985aa5c65a4735f6b3e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 8 Jun 2016 23:23:09 +0200 Subject: [PATCH 9/9] remove old vagrant scripts --- vagrant/centos-7-provision.sh | 108 -------------- vagrant/ubuntu-trusty-php7-provision.sh | 180 ------------------------ vagrant/ubuntu-trusty-provision.sh | 174 ----------------------- 3 files changed, 462 deletions(-) delete mode 100644 vagrant/centos-7-provision.sh delete mode 100755 vagrant/ubuntu-trusty-php7-provision.sh delete mode 100755 vagrant/ubuntu-trusty-provision.sh diff --git a/vagrant/centos-7-provision.sh b/vagrant/centos-7-provision.sh deleted file mode 100644 index db9d1b4f..00000000 --- a/vagrant/centos-7-provision.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# This script sets up a Nominatim installation on a CentOS 7 box. -# -# For more detailed CentOS installation instructions see also -# http://wiki.openstreetmap.org/wiki/Nominatim/Installation_on_CentOS - -## Part 1: System preparation - -## During 'vagrant provision' this script runs as root and the current -## directory is '/root' -USERNAME=vagrant - -yum update -y -yum install -y epel-release - -yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \ - make cmake gcc gcc-c++ libtool policycoreutils-python \ - php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \ - bzip2-devel proj-devel geos-devel libxml2-devel boost-devel \ - expat-devel zlib-devel - -# Create a cluster and start up postgresql. -postgresql-setup initdb -systemctl enable postgresql -systemctl start postgresql - -# We leave postgresql in its default configuration here. This is only -# suitable for small extracts. - -# Create the necessary postgres users. -sudo -u postgres createuser -s vagrant -sudo -u postgres createuser apache - -# Create the website directory. -mkdir -m 755 /var/www/html/nominatim -chown vagrant /var/www/html/nominatim - -# Set up the necessary rights on SELinux. -semanage fcontext -a -t httpd_sys_content_t "/home/vagrant/Nominatim/(website|lib|settings)(/.*)?" -semanage fcontext -a -t lib_t "/home/vagrant/Nominatim/module/nominatim.so" -semanage port -a -t http_port_t -p tcp 8089 -restorecon -R -v /home/vagrant/Nominatim - -# Configure apache site. -echo ' -Listen 8089 - - # DirectoryIndex index.html - # ErrorDocument 403 /index.html - - DocumentRoot "/var/www/html/" - - - Options FollowSymLinks MultiViews - AddType text/html .php - - -' | sudo tee /etc/httpd/conf.d/nominatim.conf > /dev/null - -# Restart apache to enable the site configuration. -systemctl enable httpd -systemctl restart httpd - -## Part 2: Nominatim installaion - -# now ideally login as $USERNAME and continue -cd /home/$USERNAME - -# If the Nominatim source is not being shared with the host, check out source. -if [ ! -d "Nominatim" ]; then - yum install -y git - sudo -H -u $USERNAME git clone --recursive https://github.com/twain47/Nominatim.git -fi - -# Configure and compile the source. -cd Nominatim -sudu -u $USERNAME mkdir build-vagrant -cd build-vagrant -sudo -u $USERNAME cmake .. -sudo -u $USERNAME make - -# Make sure that postgres has access to the nominatim library. -chmod +x /home/$USERNAME -chmod +x ./ -chmod +x ./module - -# Create customized settings suitable for this VM installation. -LOCALSETTINGS_FILE='settings/local.php' -if [[ -e "$LOCALSETTINGS_FILE" ]]; then - echo "$LOCALSETTINGS_FILE already exist, writing to settings/local-vagrant.php instead." - LOCALSETTINGS_FILE='settings/local-vagrant.php' -fi - -IP=localhost -echo " $LOCALSETTINGS_FILE - -# Install the web interface. -sudo -u $USERNAME ./utils/setup.php --create-website /var/www/html/nominatim diff --git a/vagrant/ubuntu-trusty-php7-provision.sh b/vagrant/ubuntu-trusty-php7-provision.sh deleted file mode 100755 index 6ea19a37..00000000 --- a/vagrant/ubuntu-trusty-php7-provision.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash - -# This script sets up a Nominatim installation on a Ubuntu box. -# -# For more detailed installation instructions see also -# http://wiki.openstreetmap.org/wiki/Nominatim/Installation - -## Part 1: System preparation - -## During 'vagrant provision' this script runs as root and the current -## directory is '/root' -USERNAME=vagrant - -### -### maybe create ubuntu user -### - -# if [[ ! `id -u $USERNAME` ]]; then -# useradd $USERNAME --create-home --shell /bin/bash -# -# # give sudo power -# echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USERNAME-user -# chmod 0440 /etc/sudoers.d/99-$USERNAME-user -# service sudo restart -# -# # add basic .profile -# cp -r .ssh .profile .bashrc /home/$USERNAME/ -# chown -R $USERNAME /home/$USERNAME/.* -# chgrp -R $USERNAME /home/$USERNAME/.* -# -# # now ideally login as $USERNAME and continue -# su $USERNAME -l -# fi - - -sudo apt-get update -qq -sudo apt-get upgrade -y -sudo apt-get install -y build-essential libgeos-dev libpq-dev libbz2-dev \ - libtool automake libproj-dev libboost-dev libboost-system-dev \ - libboost-filesystem-dev libboost-thread-dev libexpat-dev -sudo apt-get autoremove -y - -# get arrow-keys working in terminal (e.g. editing in vi) -echo 'stty sane' >> ~/.bash_profile -echo 'export TERM=linux' >> ~/.bash_profile -source ~/.bash_profile - - -### -### PostgreSQL 9.3 + PostGIS 2.1 -### - -sudo apt-get install -y postgresql-9.3-postgis-2.1 postgresql-contrib-9.3 postgresql-server-dev-9.3 -# already included: proj-bin libgeos-dev - -# make sure OS-authenticated users (e.g. $USERNAME) can access -sudo sed -i "s/ident/trust/" /etc/postgresql/9.3/main/pg_hba.conf -sudo sed -i "s/md5/trust/" /etc/postgresql/9.3/main/pg_hba.conf -sudo sed -i "s/peer/trust/" /etc/postgresql/9.3/main/pg_hba.conf -sudo /etc/init.d/postgresql restart - -# creates the role -sudo -u postgres createuser -s $USERNAME - - - -### -### PHP for frontend -### -sudo LC_ALL=C.UTF-8 add-apt-repository -y ppa:ondrej/php -sudo apt-get update -qq -sudo apt-get install -y apache2 -sudo apt-get install -y php7.0 php7.0-pgsql php7.0-fpm libapache2-mod-php7.0 php-pear php-db - - -# get rid of some warning -# where is the ini file? 'php --ini' -echo "date.timezone = 'Etc/UTC'" | sudo tee /etc/php/7.0/cli/conf.d/99-timezone.ini > /dev/null - - - -### -### Nominatim -### -sudo apt-get install -y libgeos-c1 libgeos++-dev libxml2-dev - -## Part 2: Nominatim installaion - -# now ideally login as $USERNAME and continue -cd /home/$USERNAME - -# If the Nominatim source is not being shared with the host, check out source. -if [ ! -d "Nominatim" ]; then - sudo apt-get install -y git - sudo -u $USERNAME git clone --recursive https://github.com/twain47/Nominatim.git -fi - -cd Nominatim - -sudo -u $USERNAME ./autogen.sh -sudo -u $USERNAME ./configure -sudo -u $USERNAME make -chmod +x ./ -chmod +x ./module - - -LOCALSETTINGS_FILE='settings/local.php' -if [[ -e "$LOCALSETTINGS_FILE" ]]; then - echo "$LOCALSETTINGS_FILE already exist, writing to settings/local-vagrant.php instead." - LOCALSETTINGS_FILE='settings/local-vagrant.php' -fi - -# IP=`curl -s http://bot.whatismyipaddress.com` -IP=localhost -echo " $LOCALSETTINGS_FILE - - - - - - - -### -### Setup Apache/website -### - -sudo -u postgres createuser -SDR www-data - -echo ' -Listen 8089 - - # DirectoryIndex index.html - # ErrorDocument 403 /index.html - - DocumentRoot "/var/www/" - - - Options FollowSymLinks MultiViews - AddType text/html .php - - -' | sudo tee /etc/apache2/sites-enabled/nominatim.conf > /dev/null - - -service apache2 graceful - - -mkdir -m 755 /var/www/nominatim -chown $USERNAME /var/www/nominatim -sudo -u $USERNAME ./utils/setup.php --create-website /var/www/nominatim - - -# if you get 'permission denied for relation word', then try -# GRANT usage ON SCHEMA public TO "www-data"; -# GRANT SELECT ON ALL TABLES IN SCHEMA public TO "www-data"; - -## -## Test suite (Python) -## https://github.com/twain47/Nominatim/tree/master/tests -## -apt-get install -y python-dev python-pip python-Levenshtein python-shapely \ - python-psycopg2 tidy python-nose python-tidylib -pip install certifi # deals with "SNI extension to TLS is not available" warning -pip install lettuce==0.2.18 six==1.7 haversine -pip install --upgrade pip setuptools - -## Test suite (PHP) -## https://github.com/twain47/Nominatim/tree/master/tests-php -wget --quiet https://phar.phpunit.de/phpunit.phar -chmod +x phpunit.phar -mv phpunit.phar /usr/local/bin/phpunit - diff --git a/vagrant/ubuntu-trusty-provision.sh b/vagrant/ubuntu-trusty-provision.sh deleted file mode 100755 index 8e35bf7d..00000000 --- a/vagrant/ubuntu-trusty-provision.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/bash - -# This script sets up a Nominatim installation on a Ubuntu box. -# -# For more detailed installation instructions see also -# http://wiki.openstreetmap.org/wiki/Nominatim/Installation - -## Part 1: System preparation - -## During 'vagrant provision' this script runs as root and the current -## directory is '/root' -USERNAME=vagrant - -### -### maybe create ubuntu user -### - -# if [[ ! `id -u $USERNAME` ]]; then -# useradd $USERNAME --create-home --shell /bin/bash -# -# # give sudo power -# echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USERNAME-user -# chmod 0440 /etc/sudoers.d/99-$USERNAME-user -# service sudo restart -# -# # add basic .profile -# cp -r .ssh .profile .bashrc /home/$USERNAME/ -# chown -R $USERNAME /home/$USERNAME/.* -# chgrp -R $USERNAME /home/$USERNAME/.* -# -# # now ideally login as $USERNAME and continue -# su $USERNAME -l -# fi - - -sudo apt-get update -qq -sudo apt-get upgrade -y -sudo apt-get install -y build-essential libgeos-dev libpq-dev libbz2-dev \ - libtool cmake libproj-dev libboost-dev libboost-system-dev \ - libboost-filesystem-dev libboost-thread-dev libexpat-dev -sudo apt-get autoremove -y - -# get arrow-keys working in terminal (e.g. editing in vi) -echo 'stty sane' >> ~/.bash_profile -echo 'export TERM=linux' >> ~/.bash_profile -source ~/.bash_profile - - -### -### PostgreSQL 9.3 + PostGIS 2.1 -### - -sudo apt-get install -y postgresql-9.3-postgis-2.1 postgresql-contrib-9.3 postgresql-server-dev-9.3 -# already included: proj-bin libgeos-dev - -# make sure OS-authenticated users (e.g. $USERNAME) can access -sudo sed -i "s/ident/trust/" /etc/postgresql/9.3/main/pg_hba.conf -sudo sed -i "s/md5/trust/" /etc/postgresql/9.3/main/pg_hba.conf -sudo sed -i "s/peer/trust/" /etc/postgresql/9.3/main/pg_hba.conf -sudo /etc/init.d/postgresql restart - -# creates the role -sudo -u postgres createuser -s $USERNAME - - - -### -### PHP for frontend -### -sudo apt-get install -y php5 php5-pgsql php-pear php-db - - -# get rid of some warning -# where is the ini file? 'php --ini' -echo "date.timezone = 'Etc/UTC'" | sudo tee /etc/php5/cli/conf.d/99-timezone.ini > /dev/null - - - -### -### Nominatim -### -sudo apt-get install -y libgeos-c1 libgeos++-dev libxml2-dev - -## Part 2: Nominatim installaion - -# now ideally login as $USERNAME and continue -cd /home/$USERNAME - -# If the Nominatim source is not being shared with the host, check out source. -if [ ! -d "Nominatim" ]; then - sudo apt-get install -y git - sudo -H -u $USERNAME git clone --recursive https://github.com/twain47/Nominatim.git -fi - -cd Nominatim -sudo -u $USERNAME mkdir build-vagrant -cd build-vagrant -sudo -u $USERNAME cmake .. -sudo -u $USERNAME make -chmod +x ./ -chmod +x ./module - - -LOCALSETTINGS_FILE='settings/local.php' -if [[ -e "$LOCALSETTINGS_FILE" ]]; then - echo "$LOCALSETTINGS_FILE already exist, writing to settings/local-vagrant.php instead." - LOCALSETTINGS_FILE='settings/local-vagrant.php' -fi - -# IP=`curl -s http://bot.whatismyipaddress.com` -IP=localhost -echo " $LOCALSETTINGS_FILE - - - - - - - -### -### Setup Apache/website -### - -sudo -u postgres createuser -SDR www-data - -echo ' -Listen 8089 - - # DirectoryIndex index.html - # ErrorDocument 403 /index.html - - DocumentRoot "/var/www/" - - - Options FollowSymLinks MultiViews - AddType text/html .php - - -' | sudo tee /etc/apache2/sites-enabled/nominatim.conf > /dev/null - - -service apache2 graceful - - -mkdir -m 755 /var/www/nominatim -chown $USERNAME /var/www/nominatim -sudo -u $USERNAME ./utils/setup.php --create-website /var/www/nominatim - - -# if you get 'permission denied for relation word', then try -# GRANT usage ON SCHEMA public TO "www-data"; -# GRANT SELECT ON ALL TABLES IN SCHEMA public TO "www-data"; - -## -## Test suite (Python) -## https://github.com/twain47/Nominatim/tree/master/tests -## -apt-get install -y python-dev python-pip python-Levenshtein python-shapely \ - python-psycopg2 tidy python-nose python-tidylib -pip install lettuce==0.2.18 six==1.7 haversine - -## Test suite (PHP) -## https://github.com/twain47/Nominatim/tree/master/tests-php -apt-get install -y phpunit - -