hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-16 05:18:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2294 from lonvia/update-actions

Browse Source 
CI: add import test against Python 3.5 and fix discovered issues
This commit is contained in:
Sarah Hoffmann
2021-04-23 23:33:15 +02:00
committed by GitHub
parent 788baafa26 9faaf3fc88
commit 5071710db7
15 changed files with 50 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/build-nominatim/action.yml vendored
View File

@@ -6,7 +6,7 @@ runs:
steps:
- name: Install prerequisites
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu python3-argparse-manpage
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu
shell: bash
- name: Download dependencies

29
.github/workflows/ci-tests.yml vendored
View File

@@ -82,7 +82,18 @@ jobs:
verbose: true
import:
runs-on: ubuntu-20.04
strategy:
matrix:
ubuntu: [18, 20]
include:
- ubuntu: 18
postgresql: 9.5
postgis: 2.5
- ubuntu: 20
postgresql: 13
postgis: 3
runs-on: ubuntu-${{ matrix.ubuntu }}.04
steps:
- uses: actions/checkout@v2
@@ -108,12 +119,24 @@ jobs:
monaco-latest.osm.pbf
key: nominatim-test-data-${{ steps.get-date.outputs.date }}
- uses: actions/setup-python@v2
with:
python-version: 3.5
if: matrix.ubuntu == 18
- uses: ./Nominatim/.github/actions/setup-postgresql
with:
postgresql-version: 13
postgis-version: 3
postgresql-version: ${{ matrix.postgresql }}
postgis-version: ${{ matrix.postgis }}
- uses: ./Nominatim/.github/actions/build-nominatim
- name: Install extra dependencies for Ubuntu 18
run: |
sudo apt-get install libicu-dev
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium
shell: bash
if: matrix.ubuntu == 18
- name: Clean installation
run: rm -rf Nominatim build
shell: bash

2
nominatim/config.py
View File

@@ -30,7 +30,7 @@ class Configuration:
self.project_dir = project_dir
self.config_dir = config_dir
self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
if project_dir is not None:
if project_dir is not None and (project_dir / '.env').is_file():
self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
# Add defaults for variables that are left empty to set the default.

2
nominatim/db/async_connection.py
View File

@@ -14,7 +14,7 @@ from psycopg2.extras import wait_select
try:
import psycopg2.errors # pylint: disable=no-name-in-module,import-error
__has_psycopg2_errors__ = True
except ModuleNotFoundError:
except ImportError:
__has_psycopg2_errors__ = False
LOG = logging.getLogger()

5
nominatim/db/status.py
View File

@@ -9,6 +9,7 @@ from nominatim.tools.exec_utils import get_url
from nominatim.errors import UsageError
LOG = logging.getLogger()
ISODATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
def compute_database_date(conn):
""" Determine the date of the database from the newest object in the
@@ -34,9 +35,9 @@ def compute_database_date(conn):
"URL used: %s", node_url)
raise UsageError("Bad API data.")
LOG.debug("Found timestamp %s", match[1])
LOG.debug("Found timestamp %s", match.group(1))
return dt.datetime.fromisoformat(match[1]).replace(tzinfo=dt.timezone.utc)
return dt.datetime.strptime(match.group(1), ISODATE_FORMAT).replace(tzinfo=dt.timezone.utc)
def set_status(conn, date, seq=None, indexed=True):

4
nominatim/tools/check_database.py
View File

@@ -85,7 +85,6 @@ def _get_indexes(conn):
'idx_placex_parent_place_id',
'idx_placex_geometry_reverse_lookuppolygon',
'idx_placex_geometry_placenode',
'idx_placex_housenumber',
'idx_osmline_parent_place_id',
'idx_osmline_parent_osm_id',
'idx_postcode_id',
@@ -100,6 +99,9 @@ def _get_indexes(conn):
'idx_location_area_country_place_id',
'idx_place_osm_unique'
))
if conn.server_version_tuple() >= (11, 0, 0):
indexes.extend(('idx_placex_housenumber',
'idx_osmline_parent_osm_id_with_hnr'))
return indexes

2
nominatim/tools/exec_utils.py
View File

@@ -99,7 +99,7 @@ def run_osm2pgsql(options):
""" Run osm2pgsql with the given options.
"""
env = get_pg_env(options['dsn'])
cmd = [options['osm2pgsql'],
cmd = [str(options['osm2pgsql']),
'--hstore', '--latlon', '--slim',
'--with-forward-dependencies', 'false',
'--log-progress', 'true',

2
nominatim/tools/replication.py
View File

@@ -13,7 +13,7 @@ from nominatim.errors import UsageError
try:
from osmium.replication.server import ReplicationServer
from osmium import WriteHandler
except ModuleNotFoundError as exc:
except ImportError as exc:
logging.getLogger().fatal("pyosmium not installed. Replication functions not available.\n"
"To install pyosmium via pip: pip3 install osmium")
raise UsageError("replication tools not available") from exc

2
nominatim/tools/special_phrases.py
View File

@@ -116,7 +116,7 @@ class SpecialPhrasesImporter():
if self.config.PHRASE_CONFIG:
settings_path = self._convert_php_settings_if_needed(self.config.PHRASE_CONFIG)
with open(settings_path, "r") as json_settings:
with settings_path.open("r") as json_settings:
settings = json.load(json_settings)
return settings['blackList'], settings['whiteList']

7
test/python/test_db_status.py
View File

@@ -19,6 +19,11 @@ OSM_NODE_DATA = """\
</osm>
"""
def iso_date(date):
return dt.datetime.strptime(date, nominatim.db.status.ISODATE_FORMAT)\
.replace(tzinfo=dt.timezone.utc)
def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_db_conn):
place_row(osm_type='N', osm_id=45673)
@@ -32,7 +37,7 @@ def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_
date = nominatim.db.status.compute_database_date(temp_db_conn)
assert requested_url == ['https://www.openstreetmap.org/api/0.6/node/45673/1']
assert date == dt.datetime.fromisoformat('2006-01-27T22:09:10').replace(tzinfo=dt.timezone.utc)
assert date == iso_date('2006-01-27T22:09:10')
def test_compute_database_broken_api(monkeypatch, status_table, place_row, temp_db_conn):

3
test/python/test_tools_replication.py
View File

@@ -41,7 +41,8 @@ def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_
temp_db_cursor.execute("SELECT * FROM import_status")
expected_date = dt.datetime.fromisoformat('2006-01-27T19:09:10').replace(tzinfo=dt.timezone.utc)
expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
.replace(tzinfo=dt.timezone.utc)
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone() == [expected_date, 234, True]

2
vagrant/Install-on-Centos-7.sh
View File

@@ -42,7 +42,7 @@
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel libicu-dev
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU argparse-manpage
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU
#

2
vagrant/Install-on-Centos-8.sh
View File

@@ -35,7 +35,7 @@
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel libicu-dev
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU argparse-manpage
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU
#

3
vagrant/Install-on-Ubuntu-18.sh
View File

@@ -30,8 +30,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php php-pgsql php-intl libicu-dev python3-pip \
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git \
python3-argparse-manpage
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
# The python-dotenv package that comes with Ubuntu 18.04 is too old, so
# install the latest version from pip:

3
vagrant/Install-on-Ubuntu-20.sh
View File

@@ -33,8 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-12 postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php php-pgsql php-intl libicu-dev python3-dotenv \
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git \
python3-argparse-manpage
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
#
# System Configuration