CI: use vagrant scripts for import tests

Use vanilla docker images of Ubuntu and leave the setup
to the vagrant scripts. Then do the usual import tests.

Also fixes a couple of issues found with the scripts

.github/workflows/ci-tests.yml

@@ -4,6 +4,7 @@ on: [ push, pull_request ]
 
 jobs:
     tests:
+        if: False
         strategy:
             matrix:
                 ubuntu: [18, 20]
@@ -120,6 +121,7 @@ jobs:
               if: matrix.ubuntu == 20
 
     import:
+        if: False
         strategy:
             matrix:
                 ubuntu: [18, 20]

.github/workflows/vagrant-scripts.yml

@@ -0,0 +1,152 @@
+name: Install scripts
+
+on: [ push, pull_request ]
+
+jobs:
+    create-archive:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v2
+              with:
+                submodules: true
+
+            - name: Get Date
+              id: get-date
+              run: |
+                  echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
+              shell: bash
+
+            - uses: actions/cache@v2
+              with:
+                  path: |
+                     data/country_grid.sql.gz
+                  key: nominatim-country-data-${{ steps.get-date.outputs.date }}
+
+            - name: Package tarball
+              run: |
+                  if [ ! -f data/country_grid.sql.gz ]; then
+                      wget --no-verbose -O data/country_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+                  fi
+                  cd ..
+                  tar cjf nominatim-src.tar.bz2 Nominatim
+                  mv nominatim-src.tar.bz2 Nominatim
+
+            - name: 'Upload Artifact'
+              uses: actions/upload-artifact@v2
+              with:
+                  name: full-source
+                  path: nominatim-src.tar.bz2
+                  retention-days: 1
+
+
+    ubuntu:
+        runs-on: ubuntu-latest
+        needs: create-archive
+
+        strategy:
+            matrix:
+                ubuntu: [18, 20]
+                include:
+                    - ubuntu: 18
+                      install_mode:
+                    - ubuntu: 20
+                      install_mode: install-apache
+
+        container:
+            image: ubuntu:${{ matrix.ubuntu }}.04
+
+        defaults:
+            run:
+                shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
+
+        steps:
+            - name: Prepare container
+              run: |
+                  export APT_LISTCHANGES_FRONTEND=none
+                  export DEBIAN_FRONTEND=noninteractive
+                  apt-get update -qq
+                  apt-get install -y git sudo wget
+                  ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
+              shell: bash
+
+            - name: Setup import user
+              run: |
+                  useradd -m nominatim -G sudo
+                  echo '%sudo   ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd
+                  echo "/home/nominatim/Nominatim/vagrant/Install-on-Ubuntu-$UBUNTU.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
+              shell: bash
+              env:
+                UBUNTU: ${{ matrix.ubuntu }}
+                INSTALL_MODE: ${{ matrix.install_mode }}
+
+            - uses: actions/download-artifact@v2
+              with:
+                  name: full-source
+                  path: /home/nominatim
+
+            - name: Get Date
+              id: get-date
+              run: |
+                  echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
+              shell: bash
+
+            - name: Install Nominatim
+              run: |
+                export USERNAME=nominatim
+                export USERHOME=/home/nominatim
+                export NOSYSTEMD=yes
+                tar xf nominatim-src.tar.bz2
+                . vagrant.sh
+              working-directory: /home/nominatim
+
+            - uses: actions/cache@v2
+              with:
+                  path: |
+                     monaco-latest.osm.pbf
+                  key: nominatim-test-data-${{ steps.get-date.outputs.date }}
+
+            - name: Prepare import environment
+              run: |
+                  if [ ! -f monaco-latest.osm.pbf ]; then
+                      wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf
+                  fi
+                  mkdir data-env data-env-reverse
+                  chown nominatim. data-env data-env-reverse
+              shell: bash
+
+            - name: Import
+              run: nominatim import --osm-file ../monaco-latest.osm.pbf
+              working-directory: data-env
+
+            - name: Import special phrases
+              run: nominatim special-phrases --import-from-wiki
+              working-directory: data-env
+
+            - name: Check full import
+              run: nominatim admin --check-database
+              working-directory: data-env
+
+            - name: Warm up database
+              run: nominatim admin --warm
+              working-directory: data-env
+
+            - name: Run update
+              run: |
+                  sudo apt-get install -y python3-pip
+                  pip3 install --user osmium
+                  nominatim replication --init
+                  NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
+              working-directory: data-env
+
+            - name: Run reverse-only import
+              run : |
+                  echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' > .env
+                  nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only --no-updates
+              working-directory: data-env-reverse
+              env:
+                  NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse
+
+            - name: Check reverse import
+              run: nominatim admin --check-database
+              working-directory: data-env-reverse

vagrant/Install-on-Ubuntu-18.sh

@@ -1,8 +1,7 @@
-#!/bin/bash
+#!/bin/bash -e
 #
 # hacks for broken vagrant box      #DOCS:
 sudo rm -f /var/lib/dpkg/lock       #DOCS:
-sudo update-locale LANG=en_US.UTF-8 #DOCS:
 export APT_LISTCHANGES_FRONTEND=none #DOCS:
 export DEBIAN_FRONTEND=noninteractive #DOCS:
 
@@ -15,10 +14,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 #
 # These instructions expect that you have a freshly installed Ubuntu 18.04.
 #
-# Make sure all packages are are up-to-date by running:
+# Make sure all packages are up-to-date by running:
 #
 
-    sudo apt-get -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" --force-yes -fuy install grub-pc #DOCS:
     sudo apt update -qq
 
 # Now you can install all packages needed for Nominatim:
@@ -30,12 +28,12 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-server-dev-10 postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php php-pgsql php-intl libicu-dev python3-pip \
-                        python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
+                        python3-psutil python3-jinja2 python3-icu git
 
 # Some of the Python packages that come with Ubuntu 18.04 are too old, so
 # install the latest version from pip:
 
-    pip3 install python-dotenv datrie pyyaml
+    pip3 install --user python-dotenv datrie pyyaml psycopg2
 
 #
 # System Configuration
@@ -60,8 +58,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # To be able to copy and paste instructions from this manual, export
 # user name and home directory now like this:
 #
+if [ "x$USERNAME" == "x" ]; then   #DOCS:
     export USERNAME=vagrant        #DOCS:    export USERNAME=nominatim
     export USERHOME=/home/vagrant  #DOCS:    export USERHOME=/srv/nominatim
+fi                                 #DOCS:
 #
 # **Never, ever run the installation as a root user.** You have been warned.
 #
@@ -79,7 +79,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 #
 # Restart the postgresql service after updating this config file.
 
-    sudo systemctl restart postgresql
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo pg_ctlcluster 10 main start
+else                                  #DOCS:
+    sudo systemctl restart postgresql #DOCS:
+fi                                    #DOCS:
 
 #
 # Finally, we need to add two postgres users: one for the user that does
@@ -133,11 +137,13 @@ fi                                 #DOCS:
 #
 # The webserver should serve the php scripts from the website directory of your
 # [project directory](../admin/Import.md#creating-the-project-directory).
-# Therefore set up a project directory and populate the website directory:
+# This directory needs to exist when being configured.
+# Therefore set up a project directory and create the website directory:
 
     mkdir $USERHOME/nominatim-project
-    cd $USERHOME/nominatim-project
+    mkdir $USERHOME/nominatim-project/website
-    nominatim refresh --website
+
+# The import process will populate the directory later.
 #
 # Option 1: Using Apache
 # ----------------------
@@ -170,7 +176,11 @@ EOFAPACHECONF
 #
 
     sudo a2enconf nominatim
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo apache2ctl start             #DOCS:
+else                                  #DOCS:
     sudo systemctl restart apache2
+fi                                    #DOCS:
 
 # The Nominatim API is now available at `http://localhost/nominatim/`.
 
@@ -250,7 +260,12 @@ EOF_NGINX_CONF
 # Enable the configuration and restart Nginx
 #
 
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS:
+    sudo /usr/sbin/nginx &            #DOCS:
+else                                  #DOCS:
     sudo systemctl restart php7.2-fpm nginx
+fi                                    #DOCS:
 
 # The Nominatim API is now available at `http://localhost/`.
 

vagrant/Install-on-Ubuntu-20.sh

@@ -1,8 +1,7 @@
-#!/bin/bash
+#!/bin/bash -e
 #
 # hacks for broken vagrant box      #DOCS:
 sudo rm -f /var/lib/dpkg/lock       #DOCS:
-sudo update-locale LANG=en_US.UTF-8 #DOCS:
 export APT_LISTCHANGES_FRONTEND=none #DOCS:
 export DEBIAN_FRONTEND=noninteractive #DOCS:
 
@@ -14,14 +13,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 #
 # These instructions expect that you have a freshly installed Ubuntu 20.04.
 #
-# Make sure all packages are are up-to-date by running:
+# Make sure all packages are up-to-date by running:
 #
 
-#DOCS:    :::sh
-    sudo apt-get \ #DOCS:
-        -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" \ #DOCS:
-        --allow-downgrades --allow-remove-essential --allow-change-held-packages \ #DOCS:
-        -fuy install grub-pc #DOCS:
     sudo apt update -qq
 
 # Now you can install all packages needed for Nominatim:
@@ -59,8 +53,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # To be able to copy and paste instructions from this manual, export
 # user name and home directory now like this:
 #
+if [ "x$USERNAME" == "x" ]; then #DOCS:
     export USERNAME=vagrant        #DOCS:    export USERNAME=nominatim
     export USERHOME=/home/vagrant  #DOCS:    export USERHOME=/srv/nominatim
+fi                                 #DOCS:
 #
 # **Never, ever run the installation as a root user.** You have been warned.
 #
@@ -78,8 +74,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 #
 # Restart the postgresql service after updating this config file.
 
-    sudo systemctl restart postgresql
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
-
+    sudo pg_ctlcluster 12 main start
+else                                  #DOCS:
+    sudo systemctl restart postgresql #DOCS:
+fi                                    #DOCS:
 #
 # Finally, we need to add two postgres users: one for the user that does
 # the import and another for the webserver which should access the database
@@ -131,11 +130,13 @@ fi                                 #DOCS:
 #
 # The webserver should serve the php scripts from the website directory of your
 # [project directory](../admin/Import.md#creating-the-project-directory).
-# Therefore set up a project directory and populate the website directory:
+# This directory needs to exist when being configured.
+# Therefore set up a project directory and create a website directory:
 
     mkdir $USERHOME/nominatim-project
-    cd $USERHOME/nominatim-project
+    mkdir $USERHOME/nominatim-project/website
-    nominatim refresh --website
+
+# The import process will populate the directory later.
 
 #
 # Option 1: Using Apache
@@ -169,7 +170,11 @@ EOFAPACHECONF
 #
 
     sudo a2enconf nominatim
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo apache2ctl start             #DOCS:
+else                                  #DOCS:
     sudo systemctl restart apache2
+fi                                    #DOCS:
 
 # The Nominatim API is now available at `http://localhost/nominatim/`.
 
@@ -252,7 +257,12 @@ EOF_NGINX_CONF
 # Enable the configuration and restart Nginx
 #
 
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo /usr/sbin/php-fpm7.3 --nodaemonize --fpm-config /etc/php/7.3/fpm/php-fpm.conf & #DOCS:
+    sudo /usr/sbin/nginx &            #DOCS:
+else                                  #DOCS:
     sudo systemctl restart php7.4-fpm nginx
+fi                                    #DOCS:
 
 # The Nominatim API is now available at `http://localhost/`.