Compare commits

..

9 Commits

Author SHA1 Message Date
Sarah Hoffmann
1a5ceec6d0 mention php-symfony-dotenv under advanced installations 2021-10-21 14:42:25 +02:00
Sarah Hoffmann
cef82b0e3f increase minimum Postgresql to 9.5
The SQL code uses array_agg with arrays which is only supported
for Postgresql 9.5. We could provide a workaround but given that
9.3/9.4 are not tested at all there is a good chance that there
are other problems.

See #2394.
2021-07-26 10:50:37 +02:00
Sarah Hoffmann
a482637a9c docs: nominatim-ui should be installed from the release
The development version does not provide the pre-packaged
dist directory anymore.
2021-07-03 21:17:43 +02:00
Sarah Hoffmann
4de8bb6e2c adapt to release 3.7.2 2021-05-30 16:54:01 +02:00
Sarah Hoffmann
41353efc5a adapt to release 3.7.1 2021-04-26 09:44:37 +02:00
Sarah Hoffmann
3eee12598d docs: update deployment to use project directory
Fixes #2295.
2021-04-26 09:42:06 +02:00
Sarah Hoffmann
8453d88f12 remove argparse dependency for vagrant scripts
Users don't need to recreate the manpage.
2021-04-26 09:41:05 +02:00
Sarah Hoffmann
99274b13c5 add migration information for new configuration format 2021-04-08 11:04:03 +02:00
Sarah Hoffmann
084838f99a adapt to release 3.7.0 2021-04-06 22:07:32 +02:00
40 changed files with 159 additions and 335 deletions

View File

@@ -6,7 +6,7 @@ runs:
steps:
- name: Install prerequisites
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu python3-argparse-manpage
shell: bash
- name: Download dependencies

View File

@@ -14,10 +14,8 @@ runs:
steps:
- name: Remove existing PostgreSQL
run: |
sudo apt-get purge -yq postgresql*
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
sudo apt-get update -qq
sudo apt-get purge -yq postgresql*
shell: bash
- name: Install PostgreSQL

View File

@@ -69,18 +69,7 @@ jobs:
working-directory: Nominatim/test/bdd
import:
strategy:
matrix:
ubuntu: [18, 20]
include:
- ubuntu: 18
postgresql: 9.5
postgis: 2.5
- ubuntu: 20
postgresql: 13
postgis: 3
runs-on: ubuntu-${{ matrix.ubuntu }}.04
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
@@ -106,24 +95,12 @@ jobs:
monaco-latest.osm.pbf
key: nominatim-test-data-${{ steps.get-date.outputs.date }}
- uses: actions/setup-python@v2
with:
python-version: 3.5
if: matrix.ubuntu == 18
- uses: ./Nominatim/.github/actions/setup-postgresql
with:
postgresql-version: ${{ matrix.postgresql }}
postgis-version: ${{ matrix.postgis }}
postgresql-version: 13
postgis-version: 3
- uses: ./Nominatim/.github/actions/build-nominatim
- name: Install extra dependencies for Ubuntu 18
run: |
sudo apt-get install libicu-dev
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium
shell: bash
if: matrix.ubuntu == 18
- name: Clean installation
run: rm -rf Nominatim build
shell: bash
@@ -146,14 +123,10 @@ jobs:
run: nominatim special-phrases --import-from-wiki
working-directory: data-env
- name: Check full import
- name: Check import
run: nominatim admin --check-database
working-directory: data-env
- name: Warm up database
run: nominatim admin --warm
working-directory: data-env
- name: Run update
run: |
nominatim replication --init
@@ -161,11 +134,7 @@ jobs:
working-directory: data-env
- name: Run reverse-only import
run : nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only --no-updates
run : nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only
working-directory: data-env
env:
NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse
- name: Check reverse import
run: nominatim admin --check-database
working-directory: data-env

View File

@@ -20,7 +20,7 @@ project(nominatim)
set(NOMINATIM_VERSION_MAJOR 3)
set(NOMINATIM_VERSION_MINOR 7)
set(NOMINATIM_VERSION_PATCH 3)
set(NOMINATIM_VERSION_PATCH 0)
set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")

View File

@@ -1,23 +1,3 @@
3.7.3
* fix XSS vulnerability in debug view
3.7.2
* fix database check for reverse-only imports
* do not error out in status API result when import date is missing
* add array_key_last function for PHP < 7.3 (thanks to @woodpack)
* fix more url when server name is unknown (thanks to @mogita)
* commit changes to replication log table
3.7.1
* fix smaller issues with special phrases import
* add index to speed up continued indexing during import
* fix index on location_property_tiger(parent_place_id)
* make sure Python code is backward-compatible with Python 3.5
* various documentation fixes
3.7.0
* switch to dotenv for configuration file

2
Vagrantfile vendored
View File

@@ -30,7 +30,7 @@ Vagrant.configure("2") do |config|
lv.memory = 2048
lv.nested = true
if ENV['CHECKOUT'] != 'y' then
override.vm.synced_folder ".", "/home/vagrant/Nominatim", type: 'nfs'
override.vm.synced_folder ".", "/home/vagrant/Nominatim", type: 'rsync'
end
end

View File

@@ -5,6 +5,7 @@ your Nominatim database. It is assumed that you have already successfully
installed the Nominatim software itself, if not return to the
[installation page](Installation.md).
## Importing multiple regions
To import multiple regions in your database, you need to configure and run `utils/import_multiple_regions.sh` file. This script will set up the update directory which has the following structure:
@@ -27,6 +28,14 @@ update
The `sequence.state` files will contain the sequence ID, which will be used by pyosmium to get updates. The tmp folder is used for import dump.
### Installing PHP prerequisites
The scripts this section still use the old PHP utils. They require
the [Symphony dotenv](https://symfony.com/doc/4.1/components/dotenv.html)
parser to work. To install it on Ubuntu/Debian, run:
sudo apt install php-symfony-dotenv
### Configuring multiple regions
The file `import_multiple_regions.sh` needs to be edited as per your requirement:

View File

@@ -37,7 +37,7 @@ For compiling:
For running Nominatim:
* [PostgreSQL](https://www.postgresql.org) (9.3+ will work, 11+ strongly recommended)
* [PostgreSQL](https://www.postgresql.org) (9.5+ will work, 11+ strongly recommended)
* [PostGIS](https://postgis.net) (2.2+)
* [Python 3](https://www.python.org/) (3.5+)
* [Psycopg2](https://www.psycopg.org) (2.7+)

View File

@@ -17,6 +17,17 @@ breaking changes. **Please read them before running the migration.**
## 3.6.0 -> 3.7.0
### New format and name of configuration file
The configuration for an import is now saved in a `.env` file in the project
directory. This file follows the dotenv format. For more information, see
the [installation chapter](Import.md#configuration-setup-in-env).
To migrate to the new system, create a new project directory, add the `.env`
file and port your custom configuration from `settings/local.php`. Most
settings are named similar and only have received a `NOMINATIM_` prefix.
Use the default settings in `settings/env.defaults` as a reference.
### New location for data files
External data files for Wikipedia importance, postcodes etc. are no longer

View File

@@ -10,12 +10,11 @@ installation. For more details, please also have a look at the
## Installing nominatim-ui
nominatim-ui does not need any special installation, just download, configure
and run it.
Clone the source from github:
git clone https://github.com/osm-search/nominatim-ui
We provide regular releases of nominatim-ui that contain the packaged website.
They do not need any special installation. Just download, configure
and run it. Grab the latest release from
[nominatim-ui's Github release page](https://github.com/osm-search/nominatim-ui/releases)
and unpack it. You can use `nominatim-ui-x.x.x.tar.gz` or `nominatim-ui-x.x.x.zip`.
Copy the example configuration into the right place:

View File

@@ -1,4 +1,4 @@
site_name: Nominatim Documentation
site_name: Nominatim 3.7.2
theme: readthedocs
docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
site_url: https://nominatim.org

View File

@@ -127,7 +127,7 @@ class Debug
public static function printSQL($sSQL)
{
echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
}
private static function outputVar($mVar, $sPreNL)
@@ -170,12 +170,11 @@ class Debug
}
if (is_string($mVar)) {
$sOut = "'$mVar'";
} else {
$sOut = (string)$mVar;
echo "'$mVar'";
return strlen($mVar) + 2;
}
echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
return strlen($sOut);
echo (string)$mVar;
return strlen((string)$mVar);
}
}

View File

@@ -51,7 +51,7 @@ class Status
$iDataDateEpoch = $this->oDB->getOne($sSQL);
if ($iDataDateEpoch === false) {
throw new Exception('Import date is not available', 705);
throw Exception('Data date query failed '.$iDataDateEpoch->getMessage(), 705);
}
return $iDataDateEpoch;

View File

@@ -227,10 +227,3 @@ function closestHouseNumber($aRow)
return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
}
if (!function_exists('array_key_last')) {
function array_key_last(array $array)
{
if (!empty($array)) return key(array_slice($array, -1, 1, true));
}
}

View File

@@ -74,7 +74,7 @@ if (isset($_SERVER['REQUEST_SCHEME'])
.$_SERVER['HTTP_HOST'].$_SERVER['DOCUMENT_URI'].'/?'
.http_build_query($aMoreParams);
} else {
$sMoreURL = '/search.php?'.http_build_query($aMoreParams);
$sMoreURL = '/search.php'.http_build_query($aMoreParams);
}
if (CONST_Debug) exit;

View File

@@ -17,23 +17,6 @@ if ($sOutputFormat == 'json') {
try {
$oStatus = new Nominatim\Status($oDB);
$oStatus->status();
if ($sOutputFormat == 'json') {
$epoch = $oStatus->dataDate();
$aResponse = array(
'status' => 0,
'message' => 'OK',
'data_updated' => (new DateTime('@'.$epoch))->format(DateTime::RFC3339),
'software_version' => CONST_NominatimVersion
);
$sDatabaseVersion = $oStatus->databaseVersion();
if ($sDatabaseVersion) {
$aResponse['database_version'] = $sDatabaseVersion;
}
javascript_renderData($aResponse);
} else {
echo 'OK';
}
} catch (Exception $oErr) {
if ($sOutputFormat == 'json') {
$aResponse = array(
@@ -45,4 +28,25 @@ try {
header('HTTP/1.0 500 Internal Server Error');
echo 'ERROR: '.$oErr->getMessage();
}
exit;
}
if ($sOutputFormat == 'json') {
$epoch = $oStatus->dataDate();
$aResponse = array(
'status' => 0,
'message' => 'OK',
'data_updated' => (new DateTime('@'.$epoch))->format(DateTime::RFC3339),
'software_version' => CONST_NominatimVersion
);
$sDatabaseVersion = $oStatus->databaseVersion();
if ($sDatabaseVersion) {
$aResponse['database_version'] = $sDatabaseVersion;
}
javascript_renderData($aResponse);
} else {
echo 'OK';
}
exit;

View File

@@ -1,5 +1,5 @@
--index only on parent_place_id
CREATE INDEX {{sql.if_index_not_exists}} idx_location_property_tiger_parent_place_id_imp
CREATE INDEX {{sql.if_index_not_exists}} idx_location_property_tiger_place_id_imp
ON location_property_tiger_import (parent_place_id) {{db.tablespace.aux_index}};
CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_location_property_tiger_place_id_imp
ON location_property_tiger_import (place_id) {{db.tablespace.aux_index}};

View File

@@ -6,9 +6,7 @@ nominatim
[-h] {import,freeze,replication,special-phrases,add-data,index,refresh,admin,export,serve,search,reverse,lookup,details,status,transition} ...
.SH DESCRIPTION
Command\-line tools for importing, updating, administrating and
.br
querying the Nominatim database.
.br
.SH OPTIONS
@@ -71,7 +69,6 @@ usage: nominatim import [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--index-noanalyse]
Create a new Nominatim database from an OSM file.
.br
@@ -128,21 +125,13 @@ Do not perform analyse operations during index
usage: nominatim freeze [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
Make database read\-only.
.br
.br
About half of data in the Nominatim database is kept only to be able to
.br
keep the data up\-to\-date with new changes made in OpenStreetMap. This
.br
command drops all this data and only keeps the part needed for geocoding
.br
itself.
.br
.br
This command has the same effect as the `\-\-no\-updates` option for imports.
.br
@@ -171,7 +160,6 @@ usage: nominatim replication [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--socket-timeout SOCKET_TIMEOUT]
Update the database using an online replication service.
.br
@@ -225,7 +213,6 @@ usage: nominatim special-phrases [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--import-from-wiki]
Import special phrases.
.br
@@ -256,13 +243,9 @@ usage: nominatim add-data [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--use-main-api]
Add additional data from a file or an online source.
.br
.br
Data is only imported, not indexed. You need to call `nominatim index`
.br
Data is only imported, not indexed. You need to call `nominatim\-update index`
to complete the process.
.br
@@ -317,7 +300,6 @@ usage: nominatim index [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--maxrank RANK]
Reindex all new and modified data.
.br
@@ -362,11 +344,8 @@ usage: nominatim refresh [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--enable-debug-statements]
Recompute auxiliary data used by the indexing process.
.br
.br
These functions must not be run in parallel with other update commands.
.br
@@ -430,7 +409,6 @@ usage: nominatim admin [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--osm-id OSM_ID | --place-id PLACE_ID]
Analyse and maintain the database.
.br
@@ -493,7 +471,6 @@ usage: nominatim export [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--restrict-to-osm-relation ID]
Export addresses as CSV file from the database.
.br
@@ -551,19 +528,12 @@ usage: nominatim serve [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--server SERVER]
Start a simple web server for serving the API.
.br
.br
This command starts the built\-in PHP webserver to serve the website
.br
from the current project directory. This webserver is only suitable
.br
for testing and develop. Do not use it in production setups!
.br
.br
By the default, the webserver can be accessed at: http://127.0.0.1:8088
.br
@@ -602,7 +572,6 @@ usage: nominatim search [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--viewbox X1,Y1,X2,Y2] [--bounded] [--no-dedupe]
Execute API search query.
.br
@@ -713,7 +682,6 @@ usage: nominatim reverse [-h] [-q] [-v] [--project-dir DIR] [-j NUM] --lat LAT
[--polygon-threshold TOLERANCE]
Execute API reverse query.
.br
@@ -783,7 +751,6 @@ usage: nominatim lookup [-h] [-q] [-v] [--project-dir DIR] [-j NUM] --id OSMID
[--polygon-threshold TOLERANCE]
Execute API lookup query.
.br
@@ -845,7 +812,6 @@ usage: nominatim details [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--lang LANGS]
Execute API details query.
.br
@@ -919,7 +885,6 @@ usage: nominatim status [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--format {text,json}]
Execute API status query.
.br
@@ -956,7 +921,6 @@ usage: nominatim transition [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
[--tiger-data FILE]
Internal functions for code transition. Do not use.
.br

View File

@@ -116,7 +116,7 @@ class UpdateAddData:
"""\
Add additional data from a file or an online source.
Data is only imported, not indexed. You need to call `nominatim index`
Data is only imported, not indexed. You need to call `nominatim-update index`
to complete the process.
"""

View File

@@ -114,7 +114,6 @@ class UpdateReplication:
if state is not replication.UpdateState.NO_CHANGES:
status.log_status(conn, start, 'import')
batchdate, _, _ = status.get_status(conn)
conn.commit()
if state is not replication.UpdateState.NO_CHANGES and args.do_index:
index_start = dt.datetime.now(dt.timezone.utc)
@@ -126,7 +125,6 @@ class UpdateReplication:
with connect(args.config.get_libpq_dsn()) as conn:
status.set_indexed(conn, True)
status.log_status(conn, index_start, 'index')
conn.commit()
else:
index_start = None

View File

@@ -105,11 +105,11 @@ class SetupAll:
LOG.error('Wikipedia importance dump file not found. '
'Will be using default importances.')
if args.continue_at is None or args.continue_at == 'load-data':
LOG.warning('Initialise tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.truncate_data_tables(conn, args.config.MAX_WORD_FREQUENCY)
if args.continue_at is None or args.continue_at == 'load-data':
LOG.warning('Load data into placex table')
database_import.load_data(args.config.get_libpq_dsn(),
args.data_dir,
@@ -120,9 +120,6 @@ class SetupAll:
nominatim_env=args, throw_on_fail=not args.ignore_errors)
if args.continue_at is None or args.continue_at in ('load-data', 'indexing'):
if args.continue_at is not None and args.continue_at != 'load-data':
with connect(args.config.get_libpq_dsn()) as conn:
SetupAll._create_pending_index(conn, args.config.TABLESPACE_ADDRESS_INDEX)
LOG.warning('Indexing places')
indexer = Indexer(args.config.get_libpq_dsn(),
args.threads or psutil.cpu_count() or 1)
@@ -152,25 +149,3 @@ class SetupAll:
'{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))
return 0
@staticmethod
def _create_pending_index(conn, tablespace):
""" Add a supporting index for finding places still to be indexed.
This index is normally created at the end of the import process
for later updates. When indexing was partially done, then this
index can greatly improve speed going through already indexed data.
"""
if conn.index_exists('idx_placex_pendingsector'):
return
with conn.cursor() as cur:
LOG.warning('Creating support index')
if tablespace:
tablespace = 'TABLESPACE ' + tablespace
cur.execute("""CREATE INDEX idx_placex_pendingsector
ON placex USING BTREE (rank_address,geometry_sector)
{} WHERE indexed_status > 0
""".format(tablespace))
conn.commit()

View File

@@ -30,7 +30,7 @@ class Configuration:
self.project_dir = project_dir
self.config_dir = config_dir
self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
if project_dir is not None and (project_dir / '.env').is_file():
if project_dir is not None:
self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
# Add defaults for variables that are left empty to set the default.

View File

@@ -14,7 +14,7 @@ from psycopg2.extras import wait_select
try:
import psycopg2.errors # pylint: disable=no-name-in-module,import-error
__has_psycopg2_errors__ = True
except ImportError:
except ModuleNotFoundError:
__has_psycopg2_errors__ = False
LOG = logging.getLogger()

View File

@@ -9,7 +9,6 @@ from ..tools.exec_utils import get_url
from ..errors import UsageError
LOG = logging.getLogger()
ISODATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
def compute_database_date(conn):
""" Determine the date of the database from the newest object in the
@@ -35,9 +34,9 @@ def compute_database_date(conn):
"URL used: %s", node_url)
raise UsageError("Bad API data.")
LOG.debug("Found timestamp %s", match.group(1))
LOG.debug("Found timestamp %s", match[1])
return dt.datetime.strptime(match.group(1), ISODATE_FORMAT).replace(tzinfo=dt.timezone.utc)
return dt.datetime.fromisoformat(match[1]).replace(tzinfo=dt.timezone.utc)
def set_status(conn, date, seq=None, indexed=True):

View File

@@ -94,9 +94,6 @@ def _get_indexes(conn):
indexes.extend(('idx_search_name_nameaddress_vector',
'idx_search_name_name_vector',
'idx_search_name_centroid'))
if conn.server_version_tuple() >= (11, 0, 0):
indexes.extend(('idx_placex_housenumber',
'idx_osmline_parent_osm_id_with_hnr'))
if conn.table_exists('place'):
indexes.extend(('idx_placex_pendingsector',
'idx_location_area_country_place_id',

View File

@@ -18,16 +18,16 @@ def run_legacy_script(script, *args, nominatim_env=None, throw_on_fail=False):
then throw a `CalledProcessError` on a non-zero exit.
"""
cmd = ['/usr/bin/env', 'php', '-Cq',
str(nominatim_env.phplib_dir / 'admin' / script)]
nominatim_env.phplib_dir / 'admin' / script]
cmd.extend([str(a) for a in args])
env = nominatim_env.config.get_os_env()
env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir)
env['NOMINATIM_SQLDIR'] = str(nominatim_env.sqllib_dir)
env['NOMINATIM_CONFIGDIR'] = str(nominatim_env.config_dir)
env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str(nominatim_env.module_dir)
env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = nominatim_env.module_dir
if not env['NOMINATIM_OSM2PGSQL_BINARY']:
env['NOMINATIM_OSM2PGSQL_BINARY'] = str(nominatim_env.osm2pgsql_path)
env['NOMINATIM_OSM2PGSQL_BINARY'] = nominatim_env.osm2pgsql_path
proc = subprocess.run(cmd, cwd=str(nominatim_env.project_dir), env=env,
check=throw_on_fail)
@@ -99,7 +99,7 @@ def run_osm2pgsql(options):
""" Run osm2pgsql with the given options.
"""
env = get_pg_env(options['dsn'])
cmd = [str(options['osm2pgsql']),
cmd = [options['osm2pgsql'],
'--hstore', '--latlon', '--slim',
'--with-forward-dependencies', 'false',
'--log-progress', 'true',

View File

@@ -13,7 +13,7 @@ from ..errors import UsageError
try:
from osmium.replication.server import ReplicationServer
from osmium import WriteHandler
except ImportError as exc:
except ModuleNotFoundError as exc:
logging.getLogger().fatal("pyosmium not installed. Replication functions not available.\n"
"To install pyosmium via pip: pip3 install osmium")
raise UsageError("replication tools not available") from exc

View File

@@ -27,7 +27,7 @@ class SpecialPhrasesImporter():
self.black_list, self.white_list = self._load_white_and_black_lists()
#Compile the regex here to increase performances.
self.occurence_pattern = re.compile(
r'\| *([^\|]+) *\|\| *([^\|]+) *\|\| *([^\|]+) *\|\| *([^\|]+) *\|\| *([\-YN])'
r'\| ([^\|]+) \|\| ([^\|]+) \|\| ([^\|]+) \|\| ([^\|]+) \|\| ([\-YN])'
)
self.sanity_check_pattern = re.compile(r'^\w+$')
self.transliterator = Transliterator.createFromRules("special-phrases normalizer",
@@ -65,7 +65,7 @@ class SpecialPhrasesImporter():
if self.config.PHRASE_CONFIG:
settings_path = self._convert_php_settings_if_needed(self.config.PHRASE_CONFIG)
with settings_path.open("r") as json_settings:
with open(settings_path, "r") as json_settings:
settings = json.load(json_settings)
return settings['blackList'], settings['whiteList']
@@ -80,7 +80,7 @@ class SpecialPhrasesImporter():
'et', 'eu', 'fa', 'fi', 'fr', 'gl', 'hr', 'hu',
'ia', 'is', 'it', 'ja', 'mk', 'nl', 'no', 'pl',
'ps', 'pt', 'ru', 'sk', 'sl', 'sv', 'uk', 'vi']
return self.config.LANGUAGES.split(',') if self.config.LANGUAGES else default_languages
return self.config.LANGUAGES or default_languages
@staticmethod
def _get_wiki_content(lang):
@@ -102,10 +102,8 @@ class SpecialPhrasesImporter():
class_matchs = self.sanity_check_pattern.findall(phrase_class)
if len(class_matchs) < 1 or len(type_matchs) < 1:
LOG.warning("Bad class/type for language %s: %s=%s. It will not be imported",
lang, phrase_class, phrase_type)
return False
return True
raise UsageError("Bad class/type for language {}: {}={}".format(
lang, phrase_class, phrase_type))
def _process_xml_content(self, xml_content, lang):
"""

View File

@@ -61,20 +61,6 @@ def handle_threaded_sql_statements(sel, file):
except Exception as exc: # pylint: disable=broad-except
LOG.info('Wrong SQL statement: %s', exc)
def handle_unregister_connection_pool(sel, place_threads):
""" Handles unregistering pool of connections
"""
while place_threads > 0:
for key, _ in sel.select(1):
conn = key.data
sel.unregister(conn)
try:
conn.wait()
except Exception as exc: # pylint: disable=broad-except
LOG.info('Wrong SQL statement: %s', exc)
conn.close()
place_threads -= 1
def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
""" Import tiger data from directory or tar file
@@ -109,7 +95,13 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
handle_threaded_sql_statements(sel, file)
# Unregistering pool of database connections
handle_unregister_connection_pool(sel, place_threads)
while place_threads > 0:
for key, _ in sel.select(1):
conn = key.data
sel.unregister(conn)
conn.wait()
conn.close()
place_threads -= 1
if tar:
tar.close()

View File

@@ -29,14 +29,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
<pre><b>Var1:</b> <i>True</i></pre>
<pre><b>Var2:</b> <i>False</i></pre>
<pre><b>Var3:</b> 0</pre>
<pre><b>Var4:</b> &#039;String&#039;</pre>
<pre><b>Var5:</b> 0 => &#039;one&#039;
1 => &#039;two&#039;
2 => &#039;three&#039;</pre>
<pre><b>Var6:</b> &#039;key&#039; => &#039;value&#039;
&#039;key2&#039; => &#039;value2&#039;</pre>
<pre><b>Var4:</b> 'String'</pre>
<pre><b>Var5:</b> 0 => 'one'
1 => 'two'
2 => 'three'</pre>
<pre><b>Var6:</b> 'key' => 'value'
'key2' => 'value2'</pre>
<pre><b>Var7:</b> me as string</pre>
<pre><b>Var8:</b> &#039;value&#039;, &#039;value2&#039;</pre>
<pre><b>Var8:</b> 'value', 'value2'</pre>
EOT
);
@@ -56,10 +56,10 @@ EOT
public function testDebugArray()
{
$this->expectOutputString(<<<EOT
<pre><b>Arr0:</b> &#039;null&#039;</pre>
<pre><b>Arr1:</b> &#039;key1&#039; => &#039;val1&#039;
&#039;key2&#039; => &#039;val2&#039;
&#039;key3&#039; => &#039;val3&#039;</pre>
<pre><b>Arr0:</b> 'null'</pre>
<pre><b>Arr1:</b> 'key1' => 'val1'
'key2' => 'val2'
'key3' => 'val3'</pre>
EOT
);
@@ -85,12 +85,12 @@ EOT
<th><small>1</small></th>
</tr>
<tr>
<td><pre>&#039;one&#039;</pre></td>
<td><pre>&#039;two&#039;</pre></td>
<td><pre>'one'</pre></td>
<td><pre>'two'</pre></td>
</tr>
<tr>
<td><pre>&#039;three&#039;</pre></td>
<td><pre>&#039;four&#039;</pre></td>
<td><pre>'three'</pre></td>
<td><pre>'four'</pre></td>
</tr>
</table>
<b>Table4:</b>
@@ -101,9 +101,9 @@ EOT
<th><small>key3</small></th>
</tr>
<tr>
<td><pre>&#039;val1&#039;</pre></td>
<td><pre>&#039;val2&#039;</pre></td>
<td><pre>&#039;val3&#039;</pre></td>
<td><pre>'val1'</pre></td>
<td><pre>'val2'</pre></td>
<td><pre>'val3'</pre></td>
</tr>
</table>
@@ -139,18 +139,18 @@ EOT
</tr>
<tr>
<td><pre>group1</pre></td>
<td><pre>&#039;val1&#039;</pre></td>
<td><pre>&#039;val2&#039;</pre></td>
<td><pre>'val1'</pre></td>
<td><pre>'val2'</pre></td>
</tr>
<tr>
<td><pre>group1</pre></td>
<td><pre>&#039;one&#039;</pre></td>
<td><pre>&#039;two&#039;</pre></td>
<td><pre>'one'</pre></td>
<td><pre>'two'</pre></td>
</tr>
<tr>
<td><pre>group2</pre></td>
<td><pre>&#039;val1&#039;</pre></td>
<td><pre>&#039;val2&#039;</pre></td>
<td><pre>'val1'</pre></td>
<td><pre>'val2'</pre></td>
</tr>
</table>
<b>Table4:</b>
@@ -163,15 +163,15 @@ EOT
</tr>
<tr>
<td><pre>group1</pre></td>
<td><pre>&#039;val1&#039;</pre></td>
<td><pre>&#039;val2&#039;</pre></td>
<td><pre>&#039;val3&#039;</pre></td>
<td><pre>'val1'</pre></td>
<td><pre>'val2'</pre></td>
<td><pre>'val3'</pre></td>
</tr>
<tr>
<td><pre>group1</pre></td>
<td><pre>&#039;val1&#039;</pre></td>
<td><pre>&#039;val2&#039;</pre></td>
<td><pre>&#039;val3&#039;</pre></td>
<td><pre>'val1'</pre></td>
<td><pre>'val2'</pre></td>
<td><pre>'val3'</pre></td>
</tr>
</table>

View File

@@ -140,7 +140,7 @@ class ParameterParserTest extends \PHPUnit\Framework\TestCase
$this->assertSame('foo', $oParams->getSet('val1', array('foo', 'bar')));
$this->assertSame(false, $oParams->getSet('val2', array('foo', 'bar')));
//$this->assertSame(0, $oParams->getSet('val3', array('foo', 'bar')));
$this->assertSame(0, $oParams->getSet('val3', array('foo', 'bar')));
}

View File

@@ -76,11 +76,10 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
$oPhrase = new Phrase('a b c', '');
$oPhrase->computeWordSets(new TokensFullSet());
/* disabled because incompatible with newer PHPUnit
$this->assertEquals(
'(a b c),(a|b c),(a b|c),(a|b|c)',
$this->serializeSets($oPhrase->getWordSets())
);*/
);
$oPhrase = new Phrase('a b c d', '');
$oPhrase->computeWordSets(new TokensFullSet());
@@ -97,11 +96,10 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
$oPhrase->computeWordSets(new TokensFullSet());
$oPhrase->invertWordSets();
/* disabled because incompatible with newer PHPUnit
$this->assertEquals(
'(a b c),(b c|a),(c|a b),(c|b|a)',
$this->serializeSets($oPhrase->getWordSets())
);*/
);
}

View File

@@ -19,11 +19,6 @@ OSM_NODE_DATA = """\
</osm>
"""
def iso_date(date):
return dt.datetime.strptime(date, nominatim.db.status.ISODATE_FORMAT)\
.replace(tzinfo=dt.timezone.utc)
def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_db_conn):
place_row(osm_type='N', osm_id=45673)
@@ -37,7 +32,7 @@ def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_
date = nominatim.db.status.compute_database_date(temp_db_conn)
assert requested_url == ['https://www.openstreetmap.org/api/0.6/node/45673/1']
assert date == iso_date('2006-01-27T22:09:10')
assert date == dt.datetime.fromisoformat('2006-01-27T22:09:10').replace(tzinfo=dt.timezone.utc)
def test_compute_database_broken_api(monkeypatch, status_table, place_row, temp_db_conn):

View File

@@ -17,11 +17,13 @@ def test_check_sanity_class(special_phrases_importer):
If a wrong class or type is given, an UsageError should raise.
If a good class and type are given, nothing special happens.
"""
with pytest.raises(UsageError):
special_phrases_importer._check_sanity('en', '', 'type')
assert not special_phrases_importer._check_sanity('en', '', 'type')
assert not special_phrases_importer._check_sanity('en', 'class', '')
with pytest.raises(UsageError):
special_phrases_importer._check_sanity('en', 'class', '')
assert special_phrases_importer._check_sanity('en', 'class', 'type')
special_phrases_importer._check_sanity('en', 'class', 'type')
def test_load_white_and_black_lists(special_phrases_importer):
"""

View File

@@ -41,8 +41,7 @@ def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_
temp_db_cursor.execute("SELECT * FROM import_status")
expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
.replace(tzinfo=dt.timezone.utc)
expected_date = dt.datetime.fromisoformat('2006-01-27T19:09:10').replace(tzinfo=dt.timezone.utc)
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone() == [expected_date, 234, True]

View File

@@ -16,24 +16,11 @@ def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1);
INSERT INTO non_existant_table values (1);""")
sqlfile.write_text("""INSERT INTO place values (1)""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_bad_file(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.txt'
sqlfile.write_text("""Random text""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
assert temp_db_cursor.table_rows('place') == 0
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
temp_db_cursor, threads, temp_db, sql_preprocessor):
@@ -41,26 +28,10 @@ def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1);
INSERT INTO non_existant_table values (1);""")
sqlfile.write_text("""INSERT INTO place values (1)""")
tar = tarfile.open("sample.tar.gz", "w:gz")
tar.add(sqlfile)
tar.close()
tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql')
assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_bad_tarfile(dsn, src_dir, def_config, tmp_path,
temp_db_cursor, threads, temp_db, sql_preprocessor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.txt'
sqlfile.write_text("""Random text""")
tar = tarfile.open("sample.tar.gz", "w:gz")
tar.add(sqlfile)
tar.close()
tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql')
assert temp_db_cursor.table_rows('place') == 0
assert temp_db_cursor.table_rows('place') == 1

View File

@@ -113,30 +113,22 @@ sudo chown vagrant /srv/nominatim #DOCS:
# Building and Configuration
# --------------------------
#
# Get the source code from Github and change into the source directory
# Get the source for the release and unpack it
#
if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME
git clone --recursive git://github.com/openstreetmap/Nominatim.git
cd Nominatim
wget https://nominatim.org/release/Nominatim-3.7.2.tar.bz2
tar xf Nominatim-3.7.2.tar.bz2
else #DOCS:
cd $USERHOME/Nominatim #DOCS:
fi #DOCS:
# When installing the latest source from github, you also need to
# download the country grid:
if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
fi #DOCS:
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
#DOCS: :::sh
mkdir $USERHOME/build
cd $USERHOME/build
cmake $USERHOME/Nominatim
cmake $USERHOME/Nominatim-3.7.2
make
sudo make install

View File

@@ -22,6 +22,7 @@
sudo dnf -qy module disable postgresql
sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
export PATH=/usr/pgsql-12/bin/:$PATH
# Now you can install all packages needed for Nominatim:
@@ -33,9 +34,9 @@
php-pgsql php php-intl php-json libpq-devel \
bzip2-devel proj-devel boost-devel \
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel libicu-dev
expat-devel zlib-devel libicu-devel
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU
pip3 install --user psycopg2-binary python-dotenv psutil Jinja2 PyICU
#
@@ -107,21 +108,12 @@ sudo chown vagrant /srv/nominatim #DOCS:
# Building and Configuration
# --------------------------
#
# Get the source code from Github and change into the source directory
# Get the source code for the release and unpack it
#
if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME
git clone --recursive git://github.com/openstreetmap/Nominatim.git
cd Nominatim
else #DOCS:
cd $USERHOME/Nominatim #DOCS:
fi #DOCS:
# When installing the latest source from github, you also need to
# download the country grid:
if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
wget https://nominatim.org/release/Nominatim-3.7.2.tar.bz2
tar xf Nominatim-3.7.2.tar.bz2
fi #DOCS:
# The code must be built in a separate directory. Create this directory,
@@ -130,7 +122,11 @@ fi #DOCS:
#DOCS: :::sh
mkdir $USERHOME/build
cd $USERHOME/build
cmake $USERHOME/Nominatim
if [ "x$1" == "xyes" ]; then #DOCS:
cmake $USERHOME/Nominatim-3.7.2
else #DOCS:
cmake /home/vagrant/Nominatim
fi #DOCS:
make
sudo make install
@@ -179,10 +175,10 @@ sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS:
# with a web server accessible from the Internet. At a minimum the
# following SELinux labeling should be done for Nominatim:
sudo semanage fcontext -a -t httpd_sys_content_t "/usr/local/nominatim/lib/lib-php(/.*)?"
sudo semanage fcontext -a -t httpd_sys_content_t "/usr/local/lib/nominatim/lib-php(/.*)?"
sudo semanage fcontext -a -t httpd_sys_content_t "$USERHOME/nominatim-project/website(/.*)?"
sudo semanage fcontext -a -t lib_t "$USERHOME/nominatim-project/module/nominatim.so"
sudo restorecon -R -v /usr/local/lib/nominatim
sudo restorecon -R -v /usr/local/lib64/nominatim
sudo restorecon -R -v $USERHOME/nominatim-project

View File

@@ -30,7 +30,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php php-pgsql php-intl libicu-dev python3-pip \
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
python3-psycopg2 python3-psutil python3-jinja2 python3-icu
# The python-dotenv package that comes with Ubuntu 18.04 is too old, so
# install the latest version from pip:
@@ -97,29 +97,22 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# Building and Configuration
# --------------------------
#
# Get the source code from Github and change into the source directory
# Get the source code for the release and unpack it
#
if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME
git clone --recursive git://github.com/openstreetmap/Nominatim.git
cd Nominatim
wget https://nominatim.org/release/Nominatim-3.7.2.tar.bz2
tar xf Nominatim-3.7.2.tar.bz2
else #DOCS:
cd $USERHOME/Nominatim #DOCS:
fi #DOCS:
# When installing the latest source from github, you also need to
# download the country grid:
if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
fi #DOCS:
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
mkdir $USERHOME/build
cd $USERHOME/build
cmake $USERHOME/Nominatim
cmake $USERHOME/Nominatim-3.7.2
make
sudo make install

View File

@@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-12 postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php php-pgsql php-intl libicu-dev python3-dotenv \
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
python3-psycopg2 python3-psutil python3-jinja2 python3-icu
#
# System Configuration
@@ -95,29 +95,22 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# Building and Configuration
# --------------------------
#
# Get the source code from Github and change into the source directory
# Get the source code for the release and unpack it
#
if [ "x$1" == "xyes" ]; then #DOCS: :::sh
cd $USERHOME
git clone --recursive git://github.com/openstreetmap/Nominatim.git
cd Nominatim
wget https://nominatim.org/release/Nominatim-3.7.2.tar.bz2
tar xf Nominatim-3.7.2.tar.bz2
else #DOCS:
cd $USERHOME/Nominatim #DOCS:
fi #DOCS:
# When installing the latest source from github, you also need to
# download the country grid:
if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
fi #DOCS:
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
mkdir $USERHOME/build
cd $USERHOME/build
cmake $USERHOME/Nominatim
cmake $USERHOME/Nominatim-3.7.2
make
sudo make install