Merge pull request #2294 from lonvia/update-actions

CI: add import test against Python 3.5 and fix discovered issues

.github/actions/build-nominatim/action.yml

@@ -6,7 +6,7 @@ runs:
     steps:
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu python3-argparse-manpage
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu
           shell: bash
 
         - name: Download dependencies

.github/workflows/ci-tests.yml

@@ -82,7 +82,18 @@ jobs:
                 verbose: true
 
     import:
-        runs-on: ubuntu-20.04
+        strategy:
+            matrix:
+                ubuntu: [18, 20]
+                include:
+                    - ubuntu: 18
+                      postgresql: 9.5
+                      postgis: 2.5
+                    - ubuntu: 20
+                      postgresql: 13
+                      postgis: 3
+
+        runs-on: ubuntu-${{ matrix.ubuntu }}.04
 
         steps:
             - uses: actions/checkout@v2
@@ -108,12 +119,24 @@ jobs:
                      monaco-latest.osm.pbf
                   key: nominatim-test-data-${{ steps.get-date.outputs.date }}
 
+            - uses: actions/setup-python@v2
+              with:
+                python-version: 3.5
+              if: matrix.ubuntu == 18
+
             - uses: ./Nominatim/.github/actions/setup-postgresql
               with:
-                  postgresql-version: 13
+                  postgresql-version: ${{ matrix.postgresql }}
-                  postgis-version: 3
+                  postgis-version: ${{ matrix.postgis }}
             - uses: ./Nominatim/.github/actions/build-nominatim
 
+            - name: Install extra dependencies for Ubuntu 18
+              run: |
+                sudo apt-get install libicu-dev
+                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium
+              shell: bash
+              if: matrix.ubuntu == 18
+
             - name: Clean installation
               run: rm -rf Nominatim build
               shell: bash

nominatim/config.py

@@ -30,7 +30,7 @@ class Configuration:
         self.project_dir = project_dir
         self.config_dir = config_dir
         self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
-        if project_dir is not None:
+        if project_dir is not None and (project_dir / '.env').is_file():
             self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
 
         # Add defaults for variables that are left empty to set the default.

nominatim/db/async_connection.py

@@ -14,7 +14,7 @@ from psycopg2.extras import wait_select
 try:
     import psycopg2.errors # pylint: disable=no-name-in-module,import-error
     __has_psycopg2_errors__ = True
-except ModuleNotFoundError:
+except ImportError:
     __has_psycopg2_errors__ = False
 
 LOG = logging.getLogger()

nominatim/db/status.py

@@ -9,6 +9,7 @@ from nominatim.tools.exec_utils import get_url
 from nominatim.errors import UsageError
 
 LOG = logging.getLogger()
+ISODATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
 
 def compute_database_date(conn):
     """ Determine the date of the database from the newest object in the
@@ -34,9 +35,9 @@ def compute_database_date(conn):
                   "URL used: %s", node_url)
         raise UsageError("Bad API data.")
 
-    LOG.debug("Found timestamp %s", match[1])
+    LOG.debug("Found timestamp %s", match.group(1))
 
-    return dt.datetime.fromisoformat(match[1]).replace(tzinfo=dt.timezone.utc)
+    return dt.datetime.strptime(match.group(1), ISODATE_FORMAT).replace(tzinfo=dt.timezone.utc)
 
 
 def set_status(conn, date, seq=None, indexed=True):

nominatim/tools/check_database.py

@@ -85,7 +85,6 @@ def _get_indexes(conn):
                'idx_placex_parent_place_id',
                'idx_placex_geometry_reverse_lookuppolygon',
                'idx_placex_geometry_placenode',
-               'idx_placex_housenumber',
                'idx_osmline_parent_place_id',
                'idx_osmline_parent_osm_id',
                'idx_postcode_id',
@@ -100,6 +99,9 @@ def _get_indexes(conn):
                         'idx_location_area_country_place_id',
                         'idx_place_osm_unique'
                        ))
+    if conn.server_version_tuple() >= (11, 0, 0):
+        indexes.extend(('idx_placex_housenumber',
+                        'idx_osmline_parent_osm_id_with_hnr'))
 
     return indexes
 

nominatim/tools/exec_utils.py

@@ -99,7 +99,7 @@ def run_osm2pgsql(options):
     """ Run osm2pgsql with the given options.
     """
     env = get_pg_env(options['dsn'])
-    cmd = [options['osm2pgsql'],
+    cmd = [str(options['osm2pgsql']),
            '--hstore', '--latlon', '--slim',
            '--with-forward-dependencies', 'false',
            '--log-progress', 'true',

nominatim/tools/replication.py

@@ -13,7 +13,7 @@ from nominatim.errors import UsageError
 try:
     from osmium.replication.server import ReplicationServer
     from osmium import WriteHandler
-except ModuleNotFoundError as exc:
+except ImportError as exc:
     logging.getLogger().fatal("pyosmium not installed. Replication functions not available.\n"
                               "To install pyosmium via pip: pip3 install osmium")
     raise UsageError("replication tools not available") from exc

nominatim/tools/special_phrases.py

@@ -116,7 +116,7 @@ class SpecialPhrasesImporter():
         if self.config.PHRASE_CONFIG:
             settings_path = self._convert_php_settings_if_needed(self.config.PHRASE_CONFIG)
 
-        with open(settings_path, "r") as json_settings:
+        with settings_path.open("r") as json_settings:
             settings = json.load(json_settings)
         return settings['blackList'], settings['whiteList']
 

test/python/test_db_status.py

@@ -19,6 +19,11 @@ OSM_NODE_DATA = """\
 </osm>
 """
 
+def iso_date(date):
+    return dt.datetime.strptime(date, nominatim.db.status.ISODATE_FORMAT)\
+               .replace(tzinfo=dt.timezone.utc)
+
+
 def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_db_conn):
     place_row(osm_type='N', osm_id=45673)
 
@@ -32,7 +37,7 @@ def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_
     date = nominatim.db.status.compute_database_date(temp_db_conn)
 
     assert requested_url == ['https://www.openstreetmap.org/api/0.6/node/45673/1']
-    assert date == dt.datetime.fromisoformat('2006-01-27T22:09:10').replace(tzinfo=dt.timezone.utc)
+    assert date == iso_date('2006-01-27T22:09:10')
 
 
 def test_compute_database_broken_api(monkeypatch, status_table, place_row, temp_db_conn):

test/python/test_tools_replication.py

@@ -41,7 +41,8 @@ def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_
 
     temp_db_cursor.execute("SELECT * FROM import_status")
 
-    expected_date = dt.datetime.fromisoformat('2006-01-27T19:09:10').replace(tzinfo=dt.timezone.utc)
+    expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
+                        .replace(tzinfo=dt.timezone.utc)
     assert temp_db_cursor.rowcount == 1
     assert temp_db_cursor.fetchone() == [expected_date, 234, True]
 

vagrant/Install-on-Centos-7.sh

@@ -42,7 +42,7 @@
                         python3-pip python3-setuptools python3-devel \
                         expat-devel zlib-devel libicu-dev
 
-    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU argparse-manpage
+    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU
 
 
 #

vagrant/Install-on-Centos-8.sh

@@ -35,7 +35,7 @@
                         python3-pip python3-setuptools python3-devel \
                         expat-devel zlib-devel libicu-dev
 
-    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU argparse-manpage
+    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU
 
 
 #

vagrant/Install-on-Ubuntu-18.sh

@@ -30,8 +30,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-server-dev-10 postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php php-pgsql php-intl libicu-dev python3-pip \
-                        python3-psycopg2 python3-psutil python3-jinja2 python3-icu git \
+                        python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
-                        python3-argparse-manpage
 
 # The python-dotenv package that comes with Ubuntu 18.04 is too old, so
 # install the latest version from pip:

vagrant/Install-on-Ubuntu-20.sh

@@ -33,8 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-server-dev-12 postgresql-12-postgis-3 \
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php php-pgsql php-intl libicu-dev python3-dotenv \
-                        python3-psycopg2 python3-psutil python3-jinja2 python3-icu git \
+                        python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
-                        python3-argparse-manpage
 
 #
 # System Configuration