mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-03-09 11:34:07 +00:00
Compare commits
128 Commits
2e2ce2c979
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d43e95f177 | ||
|
|
e22dda2a86 | ||
|
|
639f05fecc | ||
|
|
d759b6ed00 | ||
|
|
abd5cbada6 | ||
|
|
b71543b03b | ||
|
|
c25204ce31 | ||
|
|
b43116ff52 | ||
|
|
c0f1aeea4d | ||
|
|
c2d6821f2f | ||
|
|
a115eeeb40 | ||
|
|
e93c6809a9 | ||
|
|
e8836a91bb | ||
|
|
fd3dc5aeab | ||
|
|
36a364ec25 | ||
|
|
9c2d4f4285 | ||
|
|
c81fb58b63 | ||
|
|
d7249a135b | ||
|
|
757a2a6cd8 | ||
|
|
6c00169666 | ||
|
|
f0d32501e4 | ||
|
|
3e35d7fe26 | ||
|
|
fff5858b53 | ||
|
|
2507d5a298 | ||
|
|
af9458a601 | ||
|
|
855f451a5f | ||
|
|
bf17f1d01a | ||
|
|
9ac56c2078 | ||
|
|
fbe0be9301 | ||
|
|
0249cd54da | ||
|
|
52b5337f36 | ||
|
|
53e8334206 | ||
|
|
8c3c1f0a15 | ||
|
|
c31abf58d0 | ||
|
|
d0bd42298e | ||
|
|
d1b0bcaea7 | ||
|
|
c3e8fa8c43 | ||
|
|
24ba9651ba | ||
|
|
bf5ef0140a | ||
|
|
238f3dd1d9 | ||
|
|
abd7c302f8 | ||
|
|
2197236872 | ||
|
|
2ddb19c0b0 | ||
|
|
3f14f89bdf | ||
|
|
8ed7a3875a | ||
|
|
70b9140f13 | ||
|
|
3285948130 | ||
|
|
9d0732a941 | ||
|
|
5314e6c881 | ||
|
|
2750d66470 | ||
|
|
0d423ad7a7 | ||
|
|
dd332caa4d | ||
|
|
d691cfc35d | ||
|
|
d274a5aecc | ||
|
|
35a023d133 | ||
|
|
79682a94ce | ||
|
|
aa42dc8a93 | ||
|
|
29fcd0b763 | ||
|
|
2237ce7124 | ||
|
|
58295e0643 | ||
|
|
fed64cda5a | ||
|
|
b995803c66 | ||
|
|
986d303c95 | ||
|
|
310d6e3c92 | ||
|
|
7a3ea55f3d | ||
|
|
d10d70944d | ||
|
|
73590baf15 | ||
|
|
e17d0cb5cf | ||
|
|
7a62c7d812 | ||
|
|
615804b1b3 | ||
|
|
79bbdfd55c | ||
|
|
509f59b193 | ||
|
|
f84b279540 | ||
|
|
e62811cf97 | ||
|
|
cd2f6e458b | ||
|
|
fa2a789e27 | ||
|
|
fc49a77e70 | ||
|
|
28baa34bdc | ||
|
|
151a5b64a8 | ||
|
|
6fee784c9f | ||
|
|
3db7c6d804 | ||
|
|
b2f868d2fc | ||
|
|
ae7301921a | ||
|
|
8188689765 | ||
|
|
135453e463 | ||
|
|
cc9c8963f3 | ||
|
|
c882718355 | ||
|
|
3f02a4e33b | ||
|
|
1cf5464d3a | ||
|
|
dcbfa2a3d0 | ||
|
|
5cdc6724de | ||
|
|
45972811e3 | ||
|
|
e021f558bf | ||
|
|
fcc5ce3f92 | ||
|
|
9a979b7429 | ||
|
|
6ad87db1eb | ||
|
|
f4820bed0e | ||
|
|
bf6eb01d68 | ||
|
|
f07676a376 | ||
|
|
5e2ce10fe0 | ||
|
|
58cae70596 | ||
|
|
bf0ee6685b | ||
|
|
ff1f1b06d9 | ||
|
|
67ecf5f6a0 | ||
|
|
e77a4c2f35 | ||
|
|
9fa980bca2 | ||
|
|
fe773c12b2 | ||
|
|
cc96912580 | ||
|
|
77a3ecd72d | ||
|
|
6a6a064ef7 | ||
|
|
35b42ad9ce | ||
|
|
c4dc2c862e | ||
|
|
7e44256f4a | ||
|
|
eefd0efa59 | ||
|
|
2698382552 | ||
|
|
954771a42d | ||
|
|
e47601754a | ||
|
|
2cdf2db184 | ||
|
|
5200e11f33 | ||
|
|
ba1fc5a5b8 | ||
|
|
d35a71c123 | ||
|
|
e31862b7b5 | ||
|
|
9ac5e0256d | ||
|
|
a4a2176ded | ||
|
|
f30fcdcd9d | ||
|
|
77b8e76be6 | ||
|
|
20a333dd9b | ||
|
|
084e1b8177 |
12
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
12
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
## Summary
|
||||
<!-- Describe the purpose of your pull request and, if present, link to existing issues. -->
|
||||
|
||||
## AI usage
|
||||
<!-- Please list where and to what extent AI was used. -->
|
||||
|
||||
## Contributor guidelines (mandatory)
|
||||
<!-- We only accept pull requests that follow our guidelines. A deliberate violation may result in a ban. -->
|
||||
|
||||
- [ ] I have adhered to the [coding style](https://github.com/osm-search/Nominatim/blob/master/CONTRIBUTING.md#coding-style)
|
||||
- [ ] I have [tested](https://github.com/osm-search/Nominatim/blob/master/CONTRIBUTING.md#testing) the proposed changes
|
||||
- [ ] I have [disclosed](https://github.com/osm-search/Nominatim/blob/master/CONTRIBUTING.md#using-ai-assisted-code-generators) above any use of AI to generate code, documentation, or the pull request description
|
||||
95
.github/actions/setup-postgresql-windows/action.yml
vendored
Normal file
95
.github/actions/setup-postgresql-windows/action.yml
vendored
Normal file
@@ -0,0 +1,95 @@
|
||||
name: 'Setup Postgresql and Postgis on Windows'
|
||||
|
||||
description: 'Installs PostgreSQL and PostGIS for Windows and configures it for CI tests'
|
||||
|
||||
inputs:
|
||||
postgresql-version:
|
||||
description: 'Version of PostgreSQL to install'
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
|
||||
steps:
|
||||
- name: Set up PostgreSQL variables
|
||||
shell: pwsh
|
||||
run: |
|
||||
$version = "${{ inputs.postgresql-version }}"
|
||||
$root = "C:\Program Files\PostgreSQL\$version"
|
||||
$bin = "$root\bin"
|
||||
|
||||
echo "PGROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "PGBIN=$bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
echo "$bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
- name: Decide Postgis version (Windows)
|
||||
id: postgis-ver
|
||||
shell: pwsh
|
||||
run: |
|
||||
echo "PowerShell version: ${PSVersionTable.PSVersion}"
|
||||
$PG_VERSION = Split-Path $env:PGROOT -Leaf
|
||||
$postgis_page = "https://download.osgeo.org/postgis/windows/pg$PG_VERSION"
|
||||
echo "Detecting PostGIS version from $postgis_page for PostgreSQL $PG_VERSION"
|
||||
$pgis_bundle = (Invoke-WebRequest -Uri $postgis_page -ErrorAction Stop).Links.Where({$_.href -match "^postgis.*zip$"}).href
|
||||
if (!$pgis_bundle) {
|
||||
Write-Error "Could not find latest PostGIS version in $postgis_page that would match ^postgis.*zip$ pattern"
|
||||
exit 1
|
||||
}
|
||||
$pgis_bundle = [IO.Path]::ChangeExtension($pgis_bundle, [NullString]::Value)
|
||||
$pgis_bundle_url = "$postgis_page/$pgis_bundle.zip"
|
||||
Add-Content $env:GITHUB_OUTPUT "postgis_file=$pgis_bundle"
|
||||
Add-Content $env:GITHUB_OUTPUT "postgis_bundle_url=$pgis_bundle_url"
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
C:/postgis.zip
|
||||
key: postgis-cache-${{ steps.postgis-ver.outputs.postgis_file }}
|
||||
|
||||
- name: Download postgis
|
||||
shell: pwsh
|
||||
run: |
|
||||
if (!(Test-Path "C:\postgis.zip")){(new-object net.webclient).DownloadFile($env:PGIS_BUNDLE_URL, "c:\postgis.zip")}
|
||||
if (Test-path "c:\postgis_archive"){Remove-Item "c:\postgis_archive" -Recurse -Force}
|
||||
7z x c:\postgis.zip -oc:\postgis_archive
|
||||
env:
|
||||
PGIS_BUNDLE_URL: ${{ steps.postgis-ver.outputs.postgis_bundle_url }}
|
||||
|
||||
- name: Install postgis
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Root: $PGROOT, Bin: $PGBIN"
|
||||
cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT"
|
||||
|
||||
- name: Start PostgreSQL on Windows
|
||||
run: |
|
||||
$pgService = Get-Service -Name postgresql*
|
||||
Set-Service -InputObject $pgService -Status running -StartupType automatic
|
||||
Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
|
||||
shell: pwsh
|
||||
|
||||
- name: Adapt postgresql configuration
|
||||
shell: pwsh
|
||||
env:
|
||||
PGPASSWORD: root
|
||||
run: |
|
||||
& "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET fsync = 'off';"
|
||||
& "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET synchronous_commit = 'off';"
|
||||
& "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET full_page_writes = 'off';"
|
||||
& "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET shared_buffers = '1GB';"
|
||||
& "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET port = 5432;"
|
||||
|
||||
Restart-Service -Name postgresql*
|
||||
Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
|
||||
|
||||
- name: Setup database users
|
||||
shell: pwsh
|
||||
env:
|
||||
PGPASSWORD: root
|
||||
run: |
|
||||
& "$env:PGBIN\createuser" -U postgres -S www-data
|
||||
& "$env:PGBIN\createuser" -U postgres -s runner
|
||||
|
||||
|
||||
|
||||
2
.github/actions/setup-postgresql/action.yml
vendored
2
.github/actions/setup-postgresql/action.yml
vendored
@@ -1,5 +1,7 @@
|
||||
name: 'Setup Postgresql and Postgis'
|
||||
|
||||
description: 'Installs PostgreSQL and PostGIS and configures it for CI tests'
|
||||
|
||||
inputs:
|
||||
postgresql-version:
|
||||
description: 'Version of PostgreSQL to install'
|
||||
|
||||
59
.github/workflows/ci-tests.yml
vendored
59
.github/workflows/ci-tests.yml
vendored
@@ -140,6 +140,65 @@ jobs:
|
||||
../venv/bin/python -m pytest test/bdd --nominatim-purge
|
||||
working-directory: Nominatim
|
||||
|
||||
tests-windows:
|
||||
needs: create-archive
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: full-source
|
||||
|
||||
- name: Unpack Nominatim
|
||||
run: tar xf nominatim-src.tar.bz2
|
||||
|
||||
- uses: ./Nominatim/.github/actions/setup-postgresql-windows
|
||||
with:
|
||||
postgresql-version: 17
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Install Spatialite
|
||||
run: |
|
||||
Invoke-WebRequest -Uri "https://www.gaia-gis.it/gaia-sins/windows-bin-amd64/mod_spatialite-5.1.0-win-amd64.7z" -OutFile "spatialite.7z"
|
||||
7z x spatialite.7z -o"C:\spatialite"
|
||||
echo "C:\spatialite\mod_spatialite-5.1.0-win-amd64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
- name: Install osm2pgsql
|
||||
run: |
|
||||
Invoke-WebRequest -Uri "https://osm2pgsql.org/download/windows/osm2pgsql-latest-x64.zip" -OutFile "osm2pgsql.zip"
|
||||
Expand-Archive -Path "osm2pgsql.zip" -DestinationPath "C:\osm2pgsql"
|
||||
$BinDir = Get-ChildItem -Path "C:\osm2pgsql" -Recurse -Filter "osm2pgsql.exe" | Select-Object -ExpandProperty DirectoryName | Select-Object -First 1
|
||||
if (-not $BinDir) {
|
||||
Write-Error "Could not find osm2pgsql.exe"
|
||||
exit 1
|
||||
}
|
||||
echo "$BinDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
$FullExePath = Join-Path $BinDir "osm2pgsql.exe"
|
||||
echo "NOMINATIM_OSM2PGSQL_BINARY=$FullExePath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Set UTF-8 encoding
|
||||
run: |
|
||||
echo "PYTHONUTF8=1" >> $env:GITHUB_ENV
|
||||
[System.Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
||||
|
||||
- name: Install PyICU from wheel
|
||||
run: |
|
||||
python -m pip install https://github.com/cgohlke/pyicu-build/releases/download/v2.16.0/pyicu-2.16-cp314-cp314-win_amd64.whl
|
||||
|
||||
- name: Install test prerequisites
|
||||
run: |
|
||||
python -m pip install -U pip
|
||||
python -m pip install pytest pytest-asyncio "psycopg[binary]!=3.3.0" python-dotenv pyyaml jinja2 psutil sqlalchemy pytest-bdd falcon starlette uvicorn asgi_lifespan aiosqlite osmium mwparserfromhell
|
||||
|
||||
- name: Python unit tests
|
||||
run: |
|
||||
python -m pytest test/python -k "not (import_osm or run_osm2pgsql)"
|
||||
working-directory: Nominatim
|
||||
|
||||
install:
|
||||
runs-on: ubuntu-latest
|
||||
needs: create-archive
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Bugs can be reported at https://github.com/openstreetmap/Nominatim/issues.
|
||||
Please always open a separate issue for each problem. In particular, do
|
||||
not add your bugs to closed issues. They may looks similar to you but
|
||||
not add your bugs to closed issues. They may look similar to you but
|
||||
often are completely different from the maintainer's point of view.
|
||||
|
||||
## Workflow for Pull Requests
|
||||
@@ -21,7 +21,7 @@ that you are responsible for your pull requests. You should be prepared
|
||||
to get change requests because as the maintainers we have to make sure
|
||||
that your contribution fits well with the rest of the code. Please make
|
||||
sure that you have time to react to these comments and amend the code or
|
||||
engage in a conversion. Do not expect that others will pick up your code,
|
||||
engage in a conversation. Do not expect that others will pick up your code,
|
||||
it will almost never happen.
|
||||
|
||||
Please open a separate pull request for each issue you want to address.
|
||||
@@ -38,10 +38,19 @@ description or in documentation need to
|
||||
1. clearly mark the AI-generated sections as such, for example, by
|
||||
mentioning all use of AI in the PR description, and
|
||||
2. include proof that you have run the generated code on an actual
|
||||
installation of Nominatim. Adding and excuting tests will not be
|
||||
installation of Nominatim. Adding and executing tests will not be
|
||||
sufficient. You need to show that the code actually solves the problem
|
||||
the PR claims to solve.
|
||||
|
||||
## Getting Started with Development
|
||||
|
||||
Please see the development section of the Nominatim documentation for
|
||||
|
||||
* [an architecture overview](https://nominatim.org/release-docs/develop/develop/overview/)
|
||||
and backgrounds on some of the algorithms
|
||||
* [how to set up a development environment](https://nominatim.org/release-docs/develop/develop/Development-Environment/)
|
||||
* and background on [how tests are organised](https://nominatim.org/release-docs/develop/develop/Testing/)
|
||||
|
||||
|
||||
## Coding style
|
||||
|
||||
|
||||
@@ -10,14 +10,14 @@ Nominatim. Please refer to the documentation of
|
||||
[Nginx](https://nginx.org/en/docs/) for background information on how
|
||||
to configure it.
|
||||
|
||||
!!! Note
|
||||
Throughout this page, we assume your Nominatim project directory is
|
||||
located in `/srv/nominatim-project`. If you have put it somewhere else,
|
||||
you need to adjust the commands and configuration accordingly.
|
||||
|
||||
|
||||
### Installing the required packages
|
||||
|
||||
!!! warning
|
||||
ASGI support in gunicorn requires at least version 25.0. If you need
|
||||
to work with an older version of gunicorn, please refer to
|
||||
[older Nominatim deployment documentation](https://nominatim.org/release-docs/5.2/admin/Deployment-Python/)
|
||||
to learn how to run gunicorn with uvicorn.
|
||||
|
||||
The Nominatim frontend is best run from its own virtual environment. If
|
||||
you have already created one for the database backend during the
|
||||
[installation](Installation.md#building-nominatim), you can use that. Otherwise
|
||||
@@ -37,23 +37,27 @@ cd Nominatim
|
||||
```
|
||||
|
||||
The recommended way to deploy a Python ASGI application is to run
|
||||
the ASGI runner [uvicorn](https://www.uvicorn.org/)
|
||||
together with [gunicorn](https://gunicorn.org/) HTTP server. We use
|
||||
the [gunicorn](https://gunicorn.org/) HTTP server. We use
|
||||
Falcon here as the web framework.
|
||||
|
||||
Add the necessary packages to your virtual environment:
|
||||
|
||||
``` sh
|
||||
/srv/nominatim-venv/bin/pip install falcon uvicorn gunicorn
|
||||
/srv/nominatim-venv/bin/pip install falcon gunicorn
|
||||
```
|
||||
|
||||
### Setting up Nominatim as a systemd job
|
||||
|
||||
!!! Note
|
||||
These instructions assume your Nominatim project directory is
|
||||
located in `/srv/nominatim-project`. If you have put it somewhere else,
|
||||
you need to adjust the commands and configuration accordingly.
|
||||
|
||||
Next you need to set up the service that runs the Nominatim frontend. This is
|
||||
easiest done with a systemd job.
|
||||
|
||||
First you need to tell systemd to create a socket file to be used by
|
||||
hunicorn. Create the following file `/etc/systemd/system/nominatim.socket`:
|
||||
gunicorn. Create the following file `/etc/systemd/system/nominatim.socket`:
|
||||
|
||||
``` systemd
|
||||
[Unit]
|
||||
@@ -81,10 +85,8 @@ Type=simple
|
||||
User=www-data
|
||||
Group=www-data
|
||||
WorkingDirectory=/srv/nominatim-project
|
||||
ExecStart=/srv/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker "nominatim_api.server.falcon.server:run_wsgi()"
|
||||
ExecStart=/srv/nominatim-venv/bin/gunicorn -b unix:/run/nominatim.sock -w 4 --worker-class asgi --protocol uwsgi --worker-connections 1000 "nominatim_api.server.falcon.server:run_wsgi()"
|
||||
ExecReload=/bin/kill -s HUP $MAINPID
|
||||
StandardOutput=append:/var/log/gunicorn-nominatim.log
|
||||
StandardError=inherit
|
||||
PrivateTmp=true
|
||||
TimeoutStopSec=5
|
||||
KillMode=mixed
|
||||
@@ -96,7 +98,10 @@ WantedBy=multi-user.target
|
||||
This sets up gunicorn with 4 workers (`-w 4` in ExecStart). Each worker runs
|
||||
its own Python process using
|
||||
[`NOMINATIM_API_POOL_SIZE`](../customize/Settings.md#nominatim_api_pool_size)
|
||||
connections to the database to serve requests in parallel.
|
||||
connections to the database to serve requests in parallel. The parameter
|
||||
`--worker-connections` restricts how many requests gunicorn will queue for
|
||||
each worker. This can help distribute work better when the server is under
|
||||
high load.
|
||||
|
||||
Make the new services known to systemd and start it:
|
||||
|
||||
@@ -108,13 +113,15 @@ sudo systemctl enable nominatim.service
|
||||
sudo systemctl start nominatim.service
|
||||
```
|
||||
|
||||
This sets the service up, so that Nominatim is automatically started
|
||||
This sets the service up so that Nominatim is automatically started
|
||||
on reboot.
|
||||
|
||||
### Configuring nginx
|
||||
|
||||
To make the service available to the world, you need to proxy it through
|
||||
nginx. Add the following definition to the default configuration:
|
||||
nginx. We use the binary uwsgi protocol to speed up communication
|
||||
between nginx and gunicorn. Add the following definition to the default
|
||||
configuration:
|
||||
|
||||
``` nginx
|
||||
upstream nominatim_service {
|
||||
@@ -129,11 +136,8 @@ server {
|
||||
index /search;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_redirect off;
|
||||
proxy_pass http://nominatim_service;
|
||||
uwsgi_pass nominatim_service;
|
||||
include uwsgi_params;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -20,7 +20,7 @@ functions as a tie-breaker between places with very similar primary
|
||||
importance values.
|
||||
|
||||
nominatim.org has preprocessed importance tables for the
|
||||
[primary Wikipedia rankings](https://nominatim.org/data/wikimedia-importance.sql.gz)
|
||||
[primary Wikipedia rankings](https://nominatim.org/data/wikimedia-importance.csv.gz)
|
||||
and for [secondary importance](https://nominatim.org/data/wikimedia-secondary-importance.sql.gz)
|
||||
based on Wikipedia importance of the administrative areas.
|
||||
|
||||
|
||||
@@ -11,10 +11,38 @@ The import process creates the following tables:
|
||||
|
||||
The `planet_osm_*` tables are the usual backing tables for OSM data. Note
|
||||
that Nominatim uses them to look up special relations and to find nodes on
|
||||
ways.
|
||||
ways. Apart from those the osm2pgsql import produces three tables as output.
|
||||
|
||||
The osm2pgsql import produces a single table `place` as output with the following
|
||||
columns:
|
||||
The **place_postcode** table collects postcode information that is not
|
||||
already present on an object in the place table. That is for one thing
|
||||
[postcode area relations](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dpostal_code)
|
||||
and for another objects with a postcode tag but no other tagging that
|
||||
qualifies it for inclusion into the geocoding database.
|
||||
|
||||
The table has the following fields:
|
||||
|
||||
* `osm_type` - kind of OSM object (**N** - node, **W** - way, **R** - relation)
|
||||
* `osm_id` - original OSM ID
|
||||
* `postcode` - postcode as extacted from the `postcal_code` tag
|
||||
* `country_code` - computed country code for this postcode. This field
|
||||
functions as a cache and is only computed when the table is used for
|
||||
the computation of the final postcodes.
|
||||
* `centroid` - centroid of the object
|
||||
* `geometry` - the full geometry of the area for postcode areas only
|
||||
|
||||
The **place_interpolation** table holds all
|
||||
[address interpolation lines](https://wiki.openstreetmap.org/wiki/Addresses#Interpolation)
|
||||
and has the following fields:
|
||||
|
||||
* `osm_id` - original OSM ID
|
||||
* `type` - type of interpolation as extracted from the `addr:interpolation` tag
|
||||
* `address` - any other `addr:*` tags
|
||||
* `nodes` - list of OSM nodes contained in this interpolation,
|
||||
needed to compute the involved housenumbers later
|
||||
* `geometry` - the linestring for the interpolation (in WSG84)
|
||||
|
||||
The **place** table holds all other OSM object that are interesting and
|
||||
has the following fields:
|
||||
|
||||
* `osm_type` - kind of OSM object (**N** - node, **W** - way, **R** - relation)
|
||||
* `osm_id` - original OSM ID
|
||||
@@ -65,23 +93,32 @@ additional columns:
|
||||
* `indexed_status` - processing status of the place (0 - ready, 1 - freshly inserted, 2 - needs updating, 100 - needs deletion)
|
||||
* `indexed_date` - timestamp when the place was processed last
|
||||
* `centroid` - a point feature for the place
|
||||
* `token_info` - a dummy field used to inject information from the tokenizer
|
||||
into the indexing process
|
||||
|
||||
The **location_property_osmline** table is a special table for
|
||||
[address interpolations](https://wiki.openstreetmap.org/wiki/Addresses#Using_interpolation).
|
||||
The columns have the same meaning and use as the columns with the same name in
|
||||
the placex table. Only three columns are special:
|
||||
the placex table. Only the following columns are special:
|
||||
|
||||
* `startnumber` and `endnumber` - beginning and end of the number range
|
||||
for the interpolation
|
||||
* `interpolationtype` - a string `odd`, `even` or `all` to indicate
|
||||
the interval between the numbers
|
||||
* `startnumber`, `endnumber` and `step` - beginning and end of the number range
|
||||
for the interpolation and the increment steps
|
||||
* `type` - a string to indicate the interval between the numbers as imported
|
||||
from the OSM `addr:interpolation` tag; valid values are `odd`, `even`, `all`
|
||||
or a single digit number; interpolations with other values are silently
|
||||
dropped
|
||||
|
||||
Address interpolations are always ways in OSM, which is why there is no column
|
||||
`osm_type`.
|
||||
|
||||
The **location_postcodes** table holds computed centroids of all postcodes that
|
||||
can be found in the OSM data. The meaning of the columns is again the same
|
||||
as that of the placex table.
|
||||
The **location_postcodes** table holds computed postcode assembled from the
|
||||
postcode information available in OSM. When a postcode has a postcode area
|
||||
relation, then the table stores its full geometry. For all other postcode
|
||||
the centroid is computed using the position of all OSM object that reference
|
||||
the same postoce. The `osm_id` field can be used to distinguish the two.
|
||||
When set, it refers to the OSM relation with the postcode area.
|
||||
The meaning of the columns in the table is again the same as that of the
|
||||
placex table.
|
||||
|
||||
Every place needs an address, a set of surrounding places that describe the
|
||||
location of the place. The set of address places is made up of OSM places
|
||||
|
||||
@@ -56,7 +56,7 @@ The easiest way, to handle these Python dependencies is to run your
|
||||
development from within a virtual environment.
|
||||
|
||||
```sh
|
||||
sudo apt install libsqlite3-mod-spatialite osm2pgsql \
|
||||
sudo apt install build-essential libsqlite3-mod-spatialite osm2pgsql \
|
||||
postgresql-postgis postgresql-postgis-scripts \
|
||||
pkg-config libicu-dev virtualenv
|
||||
```
|
||||
@@ -68,11 +68,11 @@ virtualenv ~/nominatim-dev-venv
|
||||
~/nominatim-dev-venv/bin/pip install\
|
||||
psutil 'psycopg[binary]' PyICU SQLAlchemy \
|
||||
python-dotenv jinja2 pyYAML \
|
||||
mkdocs 'mkdocstrings[python]' mkdocs-gen-files \
|
||||
mkdocs 'mkdocstrings[python]' mkdocs-gen-files mkdocs-material \
|
||||
pytest pytest-asyncio pytest-bdd flake8 \
|
||||
types-jinja2 types-markupsafe types-psutil types-psycopg2 \
|
||||
types-pygments types-pyyaml types-requests types-ujson \
|
||||
types-urllib3 typing-extensions unicorn falcon starlette \
|
||||
types-urllib3 typing-extensions gunicorn falcon starlette \
|
||||
uvicorn mypy osmium aiosqlite mwparserfromhell
|
||||
```
|
||||
|
||||
|
||||
@@ -35,10 +35,31 @@ map place {
|
||||
geometry => GEOMETRY
|
||||
}
|
||||
|
||||
map place_postcode {
|
||||
osm_type => CHAR(1)
|
||||
osm_id => BIGINT
|
||||
postcode => TEXT
|
||||
country_code => TEXT
|
||||
centroid => GEOMETRY
|
||||
geometry => GEOMETRY
|
||||
}
|
||||
|
||||
map place_interpolation {
|
||||
osm_id => BIGINT
|
||||
type => TEXT
|
||||
address => HSTORE
|
||||
nodes => BIGINT[]
|
||||
geometry => GEOMETRY
|
||||
}
|
||||
|
||||
|
||||
planet_osm_nodes -[hidden]> planet_osm_ways
|
||||
planet_osm_ways -[hidden]> planet_osm_rels
|
||||
planet_osm_ways -[hidden]-> place
|
||||
place -[hidden]-> place_postcode
|
||||
place -[hidden]-> place_interpolation
|
||||
|
||||
planet_osm_nodes::id <- planet_osm_ways::nodes
|
||||
planet_osm_nodes::id <- place_interpolation::nodes
|
||||
|
||||
@enduml
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 19 KiB |
@@ -29,6 +29,7 @@ map placex {
|
||||
indexed_date => TIMESTAMP
|
||||
centroid => GEOMETRY
|
||||
geometry => GEOMETRY
|
||||
token_info JSONB
|
||||
}
|
||||
|
||||
map search_name {
|
||||
@@ -51,11 +52,11 @@ map word {
|
||||
map location_property_osmline {
|
||||
place_id => BIGINT
|
||||
osm_id => BIGINT
|
||||
type => TEXT
|
||||
startnumber => INT
|
||||
endnumber => INT
|
||||
interpolationtype => TEXT
|
||||
step => int
|
||||
address => HSTORE
|
||||
partition => SMALLINT
|
||||
geometry_sector => INT
|
||||
parent_place_id => BIGINT
|
||||
country_code => VARCHAR(2)
|
||||
@@ -63,6 +64,7 @@ map location_property_osmline {
|
||||
indexed_status => SMALLINT
|
||||
indexed_date => TIMESTAMP
|
||||
linegeo => GEOMETRY
|
||||
token_info JSONB
|
||||
}
|
||||
|
||||
map place_addressline {
|
||||
@@ -78,6 +80,7 @@ map location_postcodes {
|
||||
place_id => BIGINT
|
||||
osm_id => BIGINT
|
||||
postcode => TEXT
|
||||
country_code => TEXT
|
||||
parent_place_id => BIGINT
|
||||
rank_search => SMALLINT
|
||||
indexed_status => SMALLINT
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 35 KiB After Width: | Height: | Size: 36 KiB |
@@ -13,7 +13,8 @@ for infile in VAGRANT_PATH.glob('Install-on-*.sh'):
|
||||
outfile = f"admin/{infile.stem}.md"
|
||||
title = infile.stem.replace('-', ' ')
|
||||
|
||||
with mkdocs_gen_files.open(outfile, "w") as outfd, infile.open() as infd:
|
||||
with mkdocs_gen_files.open(outfile, "w", encoding='utf-8') as outfd, \
|
||||
infile.open(encoding='utf-8') as infd:
|
||||
print("#", title, file=outfd)
|
||||
has_empty = False
|
||||
for line in infd:
|
||||
|
||||
@@ -77,7 +77,19 @@ local table_definitions = {
|
||||
indexes = {
|
||||
{ column = 'postcode', method = 'btree' }
|
||||
}
|
||||
}
|
||||
},
|
||||
place_interpolation = {
|
||||
ids = { type = 'way', id_column = 'osm_id' },
|
||||
columns = {
|
||||
{ column = 'type', type = 'text', not_null = true },
|
||||
{ column = 'address', type = 'hstore' },
|
||||
{ column = 'nodes', type = 'text', sql_type = 'bigint[]', not_null = true },
|
||||
{ column = 'geometry', type = 'linestring', projection = 'WGS84', not_null = true },
|
||||
},
|
||||
indexes = {
|
||||
{ column = 'nodes', method = 'gin' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
local insert_row = {}
|
||||
@@ -703,9 +715,24 @@ function module.process_tags(o)
|
||||
o.address['country'] = nil
|
||||
end
|
||||
|
||||
if o.address.interpolation ~= nil then
|
||||
o:write_place('place', 'houses', PlaceTransform.always)
|
||||
return
|
||||
if o.address.interpolation ~= nil and o.address.housenumber == nil
|
||||
and o.object.type == 'way' and o.object.nodes ~= nil then
|
||||
local extra_addr = nil
|
||||
for k, v in pairs(o.address) do
|
||||
if k ~= 'interpolation' then
|
||||
if extra_addr == nil then
|
||||
extra_addr = {}
|
||||
end
|
||||
extra_addr[k] = v
|
||||
end
|
||||
end
|
||||
|
||||
insert_row.place_interpolation{
|
||||
type = o.address.interpolation,
|
||||
address = extra_addr,
|
||||
nodes = '{' .. table.concat(o.object.nodes, ',') .. '}',
|
||||
geometry = o.object:as_linestring()
|
||||
}
|
||||
end
|
||||
|
||||
-- collect main keys
|
||||
@@ -728,7 +755,7 @@ function module.process_tags(o)
|
||||
}
|
||||
end
|
||||
elseif ktype == 'fallback' and o.has_name then
|
||||
fallback = {k, v, PlaceTransform.named}
|
||||
fallback = {k, v, PlaceTransform.always}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2022 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- Functions for interpreting wkipedia/wikidata tags and computing importance.
|
||||
@@ -166,7 +166,7 @@ BEGIN
|
||||
END LOOP;
|
||||
|
||||
-- Nothing? Then try with the wikidata tag.
|
||||
IF result.importance is null AND extratags ? 'wikidata' THEN
|
||||
IF extratags ? 'wikidata' THEN
|
||||
FOR match IN
|
||||
{% if 'wikimedia_importance' in db.tables %}
|
||||
SELECT * FROM wikimedia_importance
|
||||
@@ -185,18 +185,18 @@ BEGIN
|
||||
END IF;
|
||||
|
||||
-- Still nothing? Fall back to a default.
|
||||
IF result.importance is null THEN
|
||||
result.importance := 0.40001 - (rank_search::float / 75);
|
||||
END IF;
|
||||
result.importance := 0.40001 - (rank_search::float / 75);
|
||||
|
||||
{% if 'secondary_importance' in db.tables %}
|
||||
FOR match IN
|
||||
SELECT ST_Value(rast, centroid) as importance
|
||||
FROM secondary_importance
|
||||
WHERE ST_Intersects(ST_ConvexHull(rast), centroid) LIMIT 1
|
||||
FROM secondary_importance
|
||||
WHERE ST_Intersects(ST_ConvexHull(rast), centroid) LIMIT 1
|
||||
LOOP
|
||||
-- Secondary importance as tie breaker with 0.0001 weight.
|
||||
result.importance := result.importance + match.importance::float / 655350000;
|
||||
IF match.importance is not NULL THEN
|
||||
-- Secondary importance as tie breaker with 0.0001 weight.
|
||||
result.importance := result.importance + match.importance::float / 655350000;
|
||||
END IF;
|
||||
END LOOP;
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -2,11 +2,99 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2022 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- Functions for address interpolation objects in location_property_osmline.
|
||||
|
||||
CREATE OR REPLACE FUNCTION place_interpolation_insert()
|
||||
RETURNS TRIGGER
|
||||
AS $$
|
||||
DECLARE
|
||||
existing RECORD;
|
||||
existingplacex BIGINT[];
|
||||
|
||||
BEGIN
|
||||
IF NOT (NEW.type in ('odd', 'even', 'all') OR NEW.type similar to '[1-9]') THEN
|
||||
-- the new interpolation is illegal, simply remove existing entries
|
||||
DELETE FROM location_property_osmline o WHERE o.osm_id = NEW.osm_id;
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
-- Remove the place from the list of places to be deleted
|
||||
DELETE FROM place_interpolation_to_be_deleted pdel WHERE pdel.osm_id = NEW.osm_id;
|
||||
|
||||
SELECT * INTO existing FROM place_interpolation p WHERE p.osm_id = NEW.osm_id;
|
||||
|
||||
-- Get the existing entry from the interpolation table.
|
||||
SELECT array_agg(place_id) INTO existingplacex
|
||||
FROM location_property_osmline o WHERE o.osm_id = NEW.osm_id;
|
||||
|
||||
IF array_length(existingplacex, 1) is NULL THEN
|
||||
INSERT INTO location_property_osmline (osm_id, type, address, linegeo)
|
||||
VALUES (NEW.osm_id, NEW.type, NEW.address, NEW.geometry);
|
||||
ELSE
|
||||
-- Update the interpolation table:
|
||||
-- The first entry gets the original data, all other entries
|
||||
-- are removed and will be recreated on indexing.
|
||||
-- (An interpolation can be split up, if it has more than 2 address nodes)
|
||||
-- Update unconditionally here as the changes might be coming from the
|
||||
-- nodes on the interpolation.
|
||||
UPDATE location_property_osmline
|
||||
SET type = NEW.type,
|
||||
address = NEW.address,
|
||||
linegeo = NEW.geometry,
|
||||
startnumber = null,
|
||||
indexed_status = 1
|
||||
WHERE place_id = existingplacex[1];
|
||||
IF array_length(existingplacex, 1) > 1 THEN
|
||||
DELETE FROM location_property_osmline WHERE place_id = any(existingplacex[2:]);
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- need to invalidate nodes because they might copy address info
|
||||
IF NEW.address is not NULL
|
||||
AND (existing.osm_id is NULL
|
||||
OR coalesce(existing.address, ''::hstore) != NEW.address)
|
||||
THEN
|
||||
UPDATE placex SET indexed_status = 2
|
||||
WHERE osm_type = 'N' AND osm_id = ANY(NEW.nodes) AND indexed_status = 0;
|
||||
END IF;
|
||||
|
||||
-- finally update/insert place_interpolation itself
|
||||
|
||||
IF existing.osm_id is not NULL THEN
|
||||
-- Always updates as the nodes with the housenumber might be the reason
|
||||
-- for the change.
|
||||
UPDATE place_interpolation p
|
||||
SET type = NEW.type,
|
||||
address = NEW.address,
|
||||
nodes = NEW.nodes,
|
||||
geometry = NEW.geometry
|
||||
WHERE p.osm_id = NEW.osm_id;
|
||||
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION place_interpolation_delete()
|
||||
RETURNS TRIGGER
|
||||
AS $$
|
||||
DECLARE
|
||||
deferred BOOLEAN;
|
||||
BEGIN
|
||||
{% if debug %}RAISE WARNING 'Delete for interpolation %', OLD.osm_id;{% endif %}
|
||||
|
||||
INSERT INTO place_interpolation_to_be_deleted (osm_id) VALUES(OLD.osm_id);
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT)
|
||||
RETURNS HSTORE
|
||||
@@ -19,17 +107,20 @@ BEGIN
|
||||
RETURN in_address;
|
||||
END IF;
|
||||
|
||||
SELECT nodes INTO waynodes FROM planet_osm_ways WHERE id = wayid;
|
||||
FOR location IN
|
||||
SELECT placex.address, placex.osm_id FROM placex
|
||||
WHERE osm_type = 'N' and osm_id = ANY(waynodes)
|
||||
and placex.address is not null
|
||||
and (placex.address ? 'street' or placex.address ? 'place')
|
||||
and indexed_status < 100
|
||||
LOOP
|
||||
-- mark it as a derived address
|
||||
RETURN location.address || in_address || hstore('_inherited', '');
|
||||
END LOOP;
|
||||
SELECT nodes INTO waynodes FROM place_interpolation WHERE osm_id = wayid;
|
||||
|
||||
IF array_upper(waynodes, 1) IS NOT NULL THEN
|
||||
FOR location IN
|
||||
SELECT placex.address, placex.osm_id FROM placex
|
||||
WHERE osm_type = 'N' and osm_id = ANY(waynodes)
|
||||
and placex.address is not null
|
||||
and (placex.address ? 'street' or placex.address ? 'place')
|
||||
and indexed_status < 100
|
||||
LOOP
|
||||
-- mark it as a derived address
|
||||
RETURN location.address || coalesce(in_address, ''::hstore) || hstore('_inherited', '');
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN in_address;
|
||||
END;
|
||||
@@ -73,51 +164,6 @@ $$
|
||||
LANGUAGE plpgsql STABLE PARALLEL SAFE;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION reinsert_interpolation(way_id BIGINT, addr HSTORE,
|
||||
geom GEOMETRY)
|
||||
RETURNS INT
|
||||
AS $$
|
||||
DECLARE
|
||||
existing BIGINT[];
|
||||
BEGIN
|
||||
IF addr is NULL OR NOT addr ? 'interpolation'
|
||||
OR NOT (addr->'interpolation' in ('odd', 'even', 'all')
|
||||
or addr->'interpolation' similar to '[1-9]')
|
||||
THEN
|
||||
-- the new interpolation is illegal, simply remove existing entries
|
||||
DELETE FROM location_property_osmline WHERE osm_id = way_id;
|
||||
ELSE
|
||||
-- Get the existing entry from the interpolation table.
|
||||
SELECT array_agg(place_id) INTO existing
|
||||
FROM location_property_osmline WHERE osm_id = way_id;
|
||||
|
||||
IF existing IS NULL or array_length(existing, 1) = 0 THEN
|
||||
INSERT INTO location_property_osmline (osm_id, address, linegeo)
|
||||
VALUES (way_id, addr, geom);
|
||||
ELSE
|
||||
-- Update the interpolation table:
|
||||
-- The first entry gets the original data, all other entries
|
||||
-- are removed and will be recreated on indexing.
|
||||
-- (An interpolation can be split up, if it has more than 2 address nodes)
|
||||
UPDATE location_property_osmline
|
||||
SET address = addr,
|
||||
linegeo = geom,
|
||||
startnumber = null,
|
||||
indexed_status = 1
|
||||
WHERE place_id = existing[1];
|
||||
IF array_length(existing, 1) > 1 THEN
|
||||
DELETE FROM location_property_osmline
|
||||
WHERE place_id = any(existing[2:]);
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
RETURN 1;
|
||||
END;
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION osmline_insert()
|
||||
RETURNS TRIGGER
|
||||
AS $$
|
||||
@@ -128,20 +174,17 @@ BEGIN
|
||||
NEW.indexed_date := now();
|
||||
|
||||
IF NEW.indexed_status IS NULL THEN
|
||||
IF NEW.address is NULL OR NOT NEW.address ? 'interpolation'
|
||||
OR NOT (NEW.address->'interpolation' in ('odd', 'even', 'all')
|
||||
or NEW.address->'interpolation' similar to '[1-9]')
|
||||
THEN
|
||||
-- alphabetic interpolation is not supported
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
IF NOT(NEW.type in ('odd', 'even', 'all') OR NEW.type similar to '[1-9]') THEN
|
||||
-- alphabetic interpolation is not supported
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
NEW.indexed_status := 1; --STATUS_NEW
|
||||
centroid := get_center_point(NEW.linegeo);
|
||||
NEW.country_code := lower(get_country_code(centroid));
|
||||
centroid := get_center_point(NEW.linegeo);
|
||||
NEW.indexed_status := 1; --STATUS_NEW
|
||||
NEW.country_code := lower(get_country_code(centroid));
|
||||
|
||||
NEW.partition := get_partition(NEW.country_code);
|
||||
NEW.geometry_sector := geometry_sector(NEW.partition, centroid);
|
||||
NEW.partition := get_partition(NEW.country_code);
|
||||
NEW.geometry_sector := geometry_sector(NEW.partition, centroid);
|
||||
END IF;
|
||||
|
||||
RETURN NEW;
|
||||
@@ -182,32 +225,22 @@ BEGIN
|
||||
get_center_point(NEW.linegeo),
|
||||
NEW.linegeo);
|
||||
|
||||
-- Cannot find a parent street. We will not be able to display a reliable
|
||||
-- address, so drop entire interpolation.
|
||||
IF NEW.parent_place_id is NULL THEN
|
||||
DELETE FROM location_property_osmline where place_id = OLD.place_id;
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
NEW.token_info := token_strip_info(NEW.token_info);
|
||||
IF NEW.address ? '_inherited' THEN
|
||||
NEW.address := hstore('interpolation', NEW.address->'interpolation');
|
||||
NEW.address := NULL;
|
||||
END IF;
|
||||
|
||||
-- If the line was newly inserted, split the line as necessary.
|
||||
IF OLD.indexed_status = 1 THEN
|
||||
IF NEW.address->'interpolation' in ('odd', 'even') THEN
|
||||
IF NEW.parent_place_id is not NULL AND NEW.startnumber is NULL THEN
|
||||
IF NEW.type in ('odd', 'even') THEN
|
||||
NEW.step := 2;
|
||||
stepmod := CASE WHEN NEW.address->'interpolation' = 'odd' THEN 1 ELSE 0 END;
|
||||
stepmod := CASE WHEN NEW.type = 'odd' THEN 1 ELSE 0 END;
|
||||
ELSE
|
||||
NEW.step := CASE WHEN NEW.address->'interpolation' = 'all'
|
||||
THEN 1
|
||||
ELSE (NEW.address->'interpolation')::SMALLINT END;
|
||||
NEW.step := CASE WHEN NEW.type = 'all' THEN 1 ELSE (NEW.type)::SMALLINT END;
|
||||
stepmod := NULL;
|
||||
END IF;
|
||||
|
||||
SELECT nodes INTO waynodes
|
||||
FROM planet_osm_ways WHERE id = NEW.osm_id;
|
||||
SELECT nodes INTO waynodes FROM place_interpolation WHERE osm_id = NEW.osm_id;
|
||||
|
||||
IF array_upper(waynodes, 1) IS NULL THEN
|
||||
RETURN NEW;
|
||||
@@ -314,12 +347,12 @@ BEGIN
|
||||
ELSE
|
||||
INSERT INTO location_property_osmline
|
||||
(linegeo, partition, osm_id, parent_place_id,
|
||||
startnumber, endnumber, step,
|
||||
startnumber, endnumber, step, type,
|
||||
address, postcode, country_code,
|
||||
geometry_sector, indexed_status)
|
||||
VALUES (ST_ReducePrecision(sectiongeo, 0.0000001),
|
||||
NEW.partition, NEW.osm_id, NEW.parent_place_id,
|
||||
startnumber, endnumber, NEW.step,
|
||||
startnumber, endnumber, NEW.step, NEW.type,
|
||||
NEW.address, postcode,
|
||||
NEW.country_code, NEW.geometry_sector, 0);
|
||||
END IF;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2022 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TYPE IF EXISTS nearfeaturecentr CASCADE;
|
||||
@@ -123,10 +123,12 @@ BEGIN
|
||||
RETURN TRUE;
|
||||
END IF;
|
||||
|
||||
IF in_rank_search <= 4 and not in_estimate THEN
|
||||
INSERT INTO location_area_country (place_id, country_code, geometry)
|
||||
(SELECT in_place_id, in_country_code, geom
|
||||
FROM split_geometry(in_geometry) as geom);
|
||||
IF in_rank_search <= 4 THEN
|
||||
IF not in_estimate and in_country_code is not NULL THEN
|
||||
INSERT INTO location_area_country (place_id, country_code, geometry)
|
||||
(SELECT in_place_id, in_country_code, geom
|
||||
FROM split_geometry(in_geometry) as geom);
|
||||
END IF;
|
||||
RETURN TRUE;
|
||||
END IF;
|
||||
|
||||
@@ -212,7 +214,6 @@ DECLARE
|
||||
BEGIN
|
||||
{% for partition in db.partitions %}
|
||||
IF in_partition = {{ partition }} THEN
|
||||
DELETE FROM search_name_{{ partition }} values WHERE place_id = in_place_id;
|
||||
IF in_rank_address > 0 THEN
|
||||
INSERT INTO search_name_{{ partition }} (place_id, address_rank, name_vector, centroid)
|
||||
values (in_place_id, in_rank_address, in_name_vector, in_geometry);
|
||||
@@ -251,7 +252,6 @@ BEGIN
|
||||
|
||||
{% for partition in db.partitions %}
|
||||
IF in_partition = {{ partition }} THEN
|
||||
DELETE FROM location_road_{{ partition }} where place_id = in_place_id;
|
||||
INSERT INTO location_road_{{ partition }} (partition, place_id, country_code, geometry)
|
||||
values (in_partition, in_place_id, in_country_code, in_geometry);
|
||||
RETURN TRUE;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2025 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
CREATE OR REPLACE FUNCTION place_insert()
|
||||
@@ -14,7 +14,6 @@ DECLARE
|
||||
existing RECORD;
|
||||
existingplacex RECORD;
|
||||
existingline BIGINT[];
|
||||
interpol RECORD;
|
||||
BEGIN
|
||||
{% if debug %}
|
||||
RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
|
||||
@@ -55,40 +54,6 @@ BEGIN
|
||||
DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
|
||||
DELETE from import_polygon_delete where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
|
||||
|
||||
-- ---- Interpolation Lines
|
||||
|
||||
IF NEW.class='place' and NEW.type='houses'
|
||||
and NEW.osm_type='W' and ST_GeometryType(NEW.geometry) = 'ST_LineString'
|
||||
THEN
|
||||
PERFORM reinsert_interpolation(NEW.osm_id, NEW.address, NEW.geometry);
|
||||
|
||||
-- Now invalidate all address nodes on the line.
|
||||
-- They get their parent from the interpolation.
|
||||
UPDATE placex p SET indexed_status = 2
|
||||
FROM planet_osm_ways w
|
||||
WHERE w.id = NEW.osm_id and p.osm_type = 'N' and p.osm_id = any(w.nodes);
|
||||
|
||||
-- If there is already an entry in place, just update that, if necessary.
|
||||
IF existing.osm_type is not null THEN
|
||||
IF coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
|
||||
OR existing.geometry::text != NEW.geometry::text
|
||||
THEN
|
||||
UPDATE place
|
||||
SET name = NEW.name,
|
||||
address = NEW.address,
|
||||
extratags = NEW.extratags,
|
||||
admin_level = NEW.admin_level,
|
||||
geometry = NEW.geometry
|
||||
WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
|
||||
and class = NEW.class and type = NEW.type;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
|
||||
-- ---- All other place types.
|
||||
|
||||
-- When an area is changed from large to small: log and discard change
|
||||
@@ -108,29 +73,6 @@ BEGIN
|
||||
RETURN null;
|
||||
END IF;
|
||||
|
||||
-- If an address node is part of a interpolation line and changes or is
|
||||
-- newly inserted (happens when the node already existed but now gets address
|
||||
-- information), then mark the interpolation line for reparenting.
|
||||
-- (Already here, because interpolation lines are reindexed before nodes,
|
||||
-- so in the second call it would be too late.)
|
||||
IF NEW.osm_type='N'
|
||||
and coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
|
||||
THEN
|
||||
FOR interpol IN
|
||||
SELECT DISTINCT osm_id, address, geometry FROM place, planet_osm_ways w
|
||||
WHERE NEW.geometry && place.geometry
|
||||
and place.osm_type = 'W'
|
||||
and place.address ? 'interpolation'
|
||||
and exists (SELECT * FROM location_property_osmline
|
||||
WHERE osm_id = place.osm_id
|
||||
and indexed_status in (0, 2))
|
||||
and w.id = place.osm_id and NEW.osm_id = any (w.nodes)
|
||||
LOOP
|
||||
PERFORM reinsert_interpolation(interpol.osm_id, interpol.address,
|
||||
interpol.geometry);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
-- Get the existing placex entry.
|
||||
SELECT * INTO existingplacex
|
||||
FROM placex
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2025 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- Trigger functions for the placex table.
|
||||
@@ -29,6 +29,7 @@ DECLARE
|
||||
location RECORD;
|
||||
result prepare_update_info;
|
||||
extra_names HSTORE;
|
||||
default_language VARCHAR(10);
|
||||
BEGIN
|
||||
IF not p.address ? '_inherited' THEN
|
||||
result.address := p.address;
|
||||
@@ -52,12 +53,8 @@ BEGIN
|
||||
-- See if we can inherit additional address tags from an interpolation.
|
||||
-- These will become permanent.
|
||||
FOR location IN
|
||||
SELECT (address - 'interpolation'::text - 'housenumber'::text) as address
|
||||
FROM place, planet_osm_ways w
|
||||
WHERE place.osm_type = 'W' and place.address ? 'interpolation'
|
||||
and place.geometry && p.geometry
|
||||
and place.osm_id = w.id
|
||||
and p.osm_id = any(w.nodes)
|
||||
SELECT address FROM place_interpolation
|
||||
WHERE ARRAY[p.osm_id] && place_interpolation.nodes AND address is not NULL
|
||||
LOOP
|
||||
result.address := location.address || result.address;
|
||||
END LOOP;
|
||||
@@ -85,6 +82,13 @@ BEGIN
|
||||
|
||||
IF location.name is not NULL THEN
|
||||
{% if debug %}RAISE WARNING 'Names original: %, location: %', result.name, location.name;{% endif %}
|
||||
|
||||
-- Add the linked-place (e.g. city) name as a searchable placename in the default language (if any)
|
||||
default_language := get_country_language_code(location.country_code);
|
||||
IF default_language is not NULL AND location.name ? 'name' AND NOT location.name ? ('name:' || default_language) THEN
|
||||
location.name := location.name || hstore('name:' || default_language, location.name->'name');
|
||||
END IF;
|
||||
|
||||
-- Add all names from the place nodes that deviate from the name
|
||||
-- in the relation with the prefix '_place_'. Deviation means that
|
||||
-- either the value is different or a given key is missing completely
|
||||
@@ -465,7 +469,7 @@ BEGIN
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
name_vector := token_get_name_search_tokens(token_info);
|
||||
name_vector := COALESCE(token_get_name_search_tokens(token_info), '{}'::INTEGER[]);
|
||||
|
||||
-- Check if the parent covers all address terms.
|
||||
-- If not, create a search name entry with the house number as the name.
|
||||
@@ -672,7 +676,7 @@ CREATE OR REPLACE FUNCTION placex_insert()
|
||||
AS $$
|
||||
DECLARE
|
||||
postcode TEXT;
|
||||
result BOOLEAN;
|
||||
result INT;
|
||||
is_area BOOLEAN;
|
||||
country_code VARCHAR(2);
|
||||
diameter FLOAT;
|
||||
@@ -777,11 +781,12 @@ BEGIN
|
||||
|
||||
|
||||
-- add to tables for special search
|
||||
-- Note: won't work on initial import because the classtype tables
|
||||
-- do not yet exist. It won't hurt either.
|
||||
classtable := 'place_classtype_' || NEW.class || '_' || NEW.type;
|
||||
SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO result;
|
||||
IF result THEN
|
||||
SELECT count(*) INTO result
|
||||
FROM pg_tables
|
||||
WHERE classtable NOT SIMILAR TO '%\W%'
|
||||
AND tablename = classtable and schemaname = current_schema();
|
||||
IF result > 0 THEN
|
||||
EXECUTE 'INSERT INTO ' || classtable::regclass || ' (place_id, centroid) VALUES ($1,$2)'
|
||||
USING NEW.place_id, NEW.centroid;
|
||||
END IF;
|
||||
@@ -840,13 +845,15 @@ BEGIN
|
||||
|
||||
NEW.indexed_date = now();
|
||||
|
||||
{% if 'search_name' in db.tables %}
|
||||
DELETE from search_name WHERE place_id = NEW.place_id;
|
||||
{% endif %}
|
||||
result := deleteSearchName(NEW.partition, NEW.place_id);
|
||||
DELETE FROM place_addressline WHERE place_id = NEW.place_id;
|
||||
result := deleteRoad(NEW.partition, NEW.place_id);
|
||||
result := deleteLocationArea(NEW.partition, NEW.place_id, NEW.rank_search);
|
||||
IF OLD.indexed_status > 1 THEN
|
||||
{% if 'search_name' in db.tables %}
|
||||
DELETE from search_name WHERE place_id = NEW.place_id;
|
||||
{% endif %}
|
||||
result := deleteSearchName(NEW.partition, NEW.place_id);
|
||||
DELETE FROM place_addressline WHERE place_id = NEW.place_id;
|
||||
result := deleteRoad(NEW.partition, NEW.place_id);
|
||||
result := deleteLocationArea(NEW.partition, NEW.place_id, NEW.rank_search);
|
||||
END IF;
|
||||
|
||||
NEW.extratags := NEW.extratags - 'linked_place'::TEXT;
|
||||
IF NEW.extratags = ''::hstore THEN
|
||||
@@ -859,10 +866,13 @@ BEGIN
|
||||
NEW.linked_place_id := OLD.linked_place_id;
|
||||
|
||||
-- Remove linkage, if we have computed a different new linkee.
|
||||
UPDATE placex SET linked_place_id = null, indexed_status = 2
|
||||
WHERE linked_place_id = NEW.place_id
|
||||
and (linked_place is null or place_id != linked_place);
|
||||
-- update not necessary for osmline, cause linked_place_id does not exist
|
||||
IF OLD.indexed_status > 1 THEN
|
||||
UPDATE placex
|
||||
SET linked_place_id = null,
|
||||
indexed_status = CASE WHEN indexed_status = 0 THEN 2 ELSE indexed_status END
|
||||
WHERE linked_place_id = NEW.place_id
|
||||
and (linked_place is null or place_id != linked_place);
|
||||
END IF;
|
||||
|
||||
-- Compute a preliminary centroid.
|
||||
NEW.centroid := get_center_point(NEW.geometry);
|
||||
@@ -1032,7 +1042,9 @@ BEGIN
|
||||
LOOP
|
||||
UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = linked_node_id;
|
||||
{% if 'search_name' in db.tables %}
|
||||
DELETE FROM search_name WHERE place_id = linked_node_id;
|
||||
IF OLD.indexed_status > 1 THEN
|
||||
DELETE FROM search_name WHERE place_id = linked_node_id;
|
||||
END IF;
|
||||
{% endif %}
|
||||
END LOOP;
|
||||
END IF;
|
||||
@@ -1181,11 +1193,6 @@ BEGIN
|
||||
-- reset the address rank if necessary.
|
||||
UPDATE placex set linked_place_id = NEW.place_id, indexed_status = 2
|
||||
WHERE place_id = location.place_id;
|
||||
-- ensure that those places are not found anymore
|
||||
{% if 'search_name' in db.tables %}
|
||||
DELETE FROM search_name WHERE place_id = location.place_id;
|
||||
{% endif %}
|
||||
PERFORM deleteLocationArea(NEW.partition, location.place_id, NEW.rank_search);
|
||||
|
||||
SELECT wikipedia, importance
|
||||
FROM compute_importance(location.extratags, NEW.country_code,
|
||||
@@ -1196,7 +1203,7 @@ BEGIN
|
||||
IF linked_importance is not null AND
|
||||
(NEW.importance is null or NEW.importance < linked_importance)
|
||||
THEN
|
||||
NEW.importance = linked_importance;
|
||||
NEW.importance := linked_importance;
|
||||
END IF;
|
||||
ELSE
|
||||
-- No linked place? As a last resort check if the boundary is tagged with
|
||||
@@ -1238,7 +1245,7 @@ BEGIN
|
||||
LIMIT 1
|
||||
LOOP
|
||||
IF location.osm_id = NEW.osm_id THEN
|
||||
{% if debug %}RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name;{% endif %}
|
||||
{% if debug %}RAISE WARNING 'Updating names for country ''%'' with: %', NEW.country_code, NEW.name;{% endif %}
|
||||
UPDATE country_name SET derived_name = NEW.name WHERE country_code = NEW.country_code;
|
||||
END IF;
|
||||
END LOOP;
|
||||
@@ -1277,10 +1284,10 @@ BEGIN
|
||||
NEW.postcode := coalesce(token_get_postcode(NEW.token_info), NEW.postcode);
|
||||
|
||||
-- if we have a name add this to the name search table
|
||||
IF NEW.name IS NOT NULL THEN
|
||||
name_vector := token_get_name_search_tokens(NEW.token_info);
|
||||
IF array_length(name_vector, 1) is not NULL THEN
|
||||
-- Initialise the name vector using our name
|
||||
NEW.name := add_default_place_name(NEW.country_code, NEW.name);
|
||||
name_vector := token_get_name_search_tokens(NEW.token_info);
|
||||
|
||||
IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
|
||||
result := add_location(NEW.place_id, NEW.country_code, NEW.partition,
|
||||
@@ -1335,15 +1342,16 @@ CREATE OR REPLACE FUNCTION placex_delete()
|
||||
AS $$
|
||||
DECLARE
|
||||
b BOOLEAN;
|
||||
result INT;
|
||||
classtable TEXT;
|
||||
BEGIN
|
||||
-- RAISE WARNING 'placex_delete % %',OLD.osm_type,OLD.osm_id;
|
||||
|
||||
IF OLD.linked_place_id is null THEN
|
||||
update placex set linked_place_id = null, indexed_status = 2 where linked_place_id = OLD.place_id and indexed_status = 0;
|
||||
{% if debug %}RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id;{% endif %}
|
||||
update placex set linked_place_id = null where linked_place_id = OLD.place_id;
|
||||
{% if debug %}RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id;{% endif %}
|
||||
UPDATE placex
|
||||
SET linked_place_id = NULL,
|
||||
indexed_status = CASE WHEN indexed_status = 0 THEN 2 ELSE indexed_status END
|
||||
WHERE linked_place_id = OLD.place_id;
|
||||
ELSE
|
||||
update placex set indexed_status = 2 where place_id = OLD.linked_place_id and indexed_status = 0;
|
||||
END IF;
|
||||
@@ -1367,6 +1375,7 @@ BEGIN
|
||||
-- reparenting also for OSM Interpolation Lines (and for Tiger?)
|
||||
update location_property_osmline set indexed_status = 2 where indexed_status = 0 and parent_place_id = OLD.place_id;
|
||||
|
||||
UPDATE location_postcodes SET indexed_status = 2 WHERE parent_place_id = OLD.place_id;
|
||||
END IF;
|
||||
|
||||
{% if debug %}RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;{% endif %}
|
||||
@@ -1392,15 +1401,16 @@ BEGIN
|
||||
|
||||
-- remove from tables for special search
|
||||
classtable := 'place_classtype_' || OLD.class || '_' || OLD.type;
|
||||
SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO b;
|
||||
IF b THEN
|
||||
SELECT count(*) INTO result
|
||||
FROM pg_tables
|
||||
WHERE classtable NOT SIMILAR TO '%\W%'
|
||||
AND tablename = classtable and schemaname = current_schema();
|
||||
|
||||
IF result > 0 THEN
|
||||
EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id;
|
||||
END IF;
|
||||
|
||||
{% if debug %}RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id;{% endif %}
|
||||
|
||||
UPDATE location_postcodes SET indexed_status = 2 WHERE parent_place_id = OLD.place_id;
|
||||
|
||||
RETURN OLD;
|
||||
|
||||
END;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2025 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- Assorted helper functions for the triggers.
|
||||
@@ -46,13 +46,13 @@ DECLARE
|
||||
r INTEGER[];
|
||||
BEGIN
|
||||
IF array_upper(a, 1) IS NULL THEN
|
||||
RETURN b;
|
||||
RETURN COALESCE(b, '{}'::INTEGER[]);
|
||||
END IF;
|
||||
IF array_upper(b, 1) IS NULL THEN
|
||||
RETURN a;
|
||||
RETURN COALESCE(a, '{}'::INTEGER[]);
|
||||
END IF;
|
||||
r := a;
|
||||
FOR i IN 1..array_upper(b, 1) LOOP
|
||||
FOR i IN 1..array_upper(b, 1) LOOP
|
||||
IF NOT (ARRAY[b[i]] <@ r) THEN
|
||||
r := r || b[i];
|
||||
END IF;
|
||||
@@ -153,8 +153,7 @@ BEGIN
|
||||
IF ST_GeometryType(geom) in ('ST_Polygon','ST_MultiPolygon') THEN
|
||||
SELECT min(postcode), count(*) FROM
|
||||
(SELECT postcode FROM location_postcodes
|
||||
WHERE geom && location_postcodes.geometry -- want to use the index
|
||||
AND ST_Contains(geom, location_postcodes.centroid)
|
||||
WHERE ST_Contains(geom, location_postcodes.centroid)
|
||||
AND country_code = country
|
||||
LIMIT 2) sub
|
||||
INTO outcode, cnt;
|
||||
@@ -368,8 +367,6 @@ CREATE OR REPLACE FUNCTION add_location(place_id BIGINT, country_code varchar(2)
|
||||
DECLARE
|
||||
postcode TEXT;
|
||||
BEGIN
|
||||
PERFORM deleteLocationArea(partition, place_id, rank_search);
|
||||
|
||||
-- add postcode only if it contains a single entry, i.e. ignore postcode lists
|
||||
postcode := NULL;
|
||||
IF in_postcode is not null AND in_postcode not similar to '%(,|;)%' THEN
|
||||
@@ -627,18 +624,22 @@ BEGIN
|
||||
and placex.type = place_to_be_deleted.type
|
||||
and not deferred;
|
||||
|
||||
-- Mark for delete in interpolations
|
||||
UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted
|
||||
WHERE place_to_be_deleted.osm_type = 'W'
|
||||
and place_to_be_deleted.class = 'place'
|
||||
and place_to_be_deleted.type = 'houses'
|
||||
and location_property_osmline.osm_id = place_to_be_deleted.osm_id
|
||||
and not deferred;
|
||||
-- Clear todo list.
|
||||
TRUNCATE TABLE place_to_be_deleted;
|
||||
|
||||
-- Clear todo list.
|
||||
TRUNCATE TABLE place_to_be_deleted;
|
||||
-- delete from place_interpolation table
|
||||
ALTER TABLE place_interpolation DISABLE TRIGGER place_interpolation_before_delete;
|
||||
DELETE FROM place_interpolation p USING place_interpolation_to_be_deleted d
|
||||
WHERE p.osm_id = d.osm_id;
|
||||
ALTER TABLE place_interpolation ENABLE TRIGGER place_interpolation_before_delete;
|
||||
|
||||
RETURN NULL;
|
||||
UPDATE location_property_osmline o SET indexed_status = 100
|
||||
FROM place_interpolation_to_be_deleted d
|
||||
WHERE o.osm_id = d.osm_id;
|
||||
|
||||
TRUNCATE TABLE place_interpolation_to_be_deleted;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
47
lib-sql/grants.sql
Normal file
47
lib-sql/grants.sql
Normal file
@@ -0,0 +1,47 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
--
|
||||
-- Grant read-only access to the web user for all Nominatim tables.
|
||||
|
||||
-- Core tables
|
||||
GRANT SELECT ON import_status TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON country_name TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON nominatim_properties TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
-- Location tables
|
||||
GRANT SELECT ON location_property_tiger TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON location_property_osmline TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON location_postcodes TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
-- Search tables
|
||||
{% if not db.reverse_only %}
|
||||
GRANT SELECT ON search_name TO "{{config.DATABASE_WEBUSER}}";
|
||||
{% endif %}
|
||||
|
||||
-- Main place tables
|
||||
GRANT SELECT ON placex TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON place_addressline TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON placex_entrance TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
-- Error/delete tracking tables
|
||||
GRANT SELECT ON import_polygon_error TO "{{config.DATABASE_WEBUSER}}";
|
||||
GRANT SELECT ON import_polygon_delete TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
-- Country grid
|
||||
GRANT SELECT ON country_osm_grid TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
-- Tokenizer tables (word table)
|
||||
{% if 'word' in db.tables %}
|
||||
GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}";
|
||||
{% endif %}
|
||||
|
||||
-- Special phrase tables
|
||||
{% for table in db.tables %}
|
||||
{% if table.startswith('place_classtype_') %}
|
||||
GRANT SELECT ON {{ table }} TO "{{config.DATABASE_WEBUSER}}";
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2025 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- Indices used only during search and update.
|
||||
@@ -67,11 +67,15 @@ CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id
|
||||
---
|
||||
-- Table needed for running updates with osm2pgsql on place.
|
||||
CREATE TABLE IF NOT EXISTS place_to_be_deleted (
|
||||
osm_type CHAR(1),
|
||||
osm_id BIGINT,
|
||||
class TEXT,
|
||||
type TEXT,
|
||||
deferred BOOLEAN
|
||||
osm_type CHAR(1) NOT NULL,
|
||||
osm_id BIGINT NOT NULL,
|
||||
class TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
deferred BOOLEAN NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS place_interpolation_to_be_deleted (
|
||||
osm_id BIGINT NOT NULL
|
||||
);
|
||||
---
|
||||
CREATE INDEX IF NOT EXISTS idx_location_postcodes_parent_place_id
|
||||
|
||||
@@ -2,36 +2,48 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2022 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
drop table IF EXISTS search_name_blank CASCADE;
|
||||
CREATE TABLE search_name_blank (
|
||||
place_id BIGINT,
|
||||
address_rank smallint,
|
||||
name_vector integer[],
|
||||
centroid GEOMETRY(Geometry, 4326)
|
||||
place_id BIGINT NOT NULL,
|
||||
address_rank smallint NOT NULL,
|
||||
name_vector integer[] NOT NULL,
|
||||
centroid GEOMETRY(Geometry, 4326) NOT NULL
|
||||
);
|
||||
|
||||
|
||||
{% for partition in db.partitions %}
|
||||
CREATE TABLE location_area_large_{{ partition }} () INHERITS (location_area_large) {{db.tablespace.address_data}};
|
||||
CREATE INDEX idx_location_area_large_{{ partition }}_place_id ON location_area_large_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_location_area_large_{{ partition }}_geometry ON location_area_large_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_location_area_large_{{ partition }}_place_id
|
||||
ON location_area_large_{{ partition }}
|
||||
USING BTREE (place_id) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_location_area_large_{{ partition }}_geometry
|
||||
ON location_area_large_{{ partition }}
|
||||
USING GIST (geometry) {{db.tablespace.address_index}};
|
||||
|
||||
CREATE TABLE search_name_{{ partition }} () INHERITS (search_name_blank) {{db.tablespace.address_data}};
|
||||
CREATE INDEX idx_search_name_{{ partition }}_place_id ON search_name_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_search_name_{{ partition }}_centroid_street ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 26 and 27;
|
||||
CREATE INDEX idx_search_name_{{ partition }}_centroid_place ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 2 and 25;
|
||||
CREATE UNIQUE INDEX idx_search_name_{{ partition }}_place_id
|
||||
ON search_name_{{ partition }}
|
||||
USING BTREE (place_id) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_search_name_{{ partition }}_centroid_street
|
||||
ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}}
|
||||
WHERE address_rank between 26 and 27;
|
||||
CREATE INDEX idx_search_name_{{ partition }}_centroid_place
|
||||
ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}}
|
||||
WHERE address_rank between 2 and 25;
|
||||
|
||||
DROP TABLE IF EXISTS location_road_{{ partition }};
|
||||
CREATE TABLE location_road_{{ partition }} (
|
||||
place_id BIGINT,
|
||||
partition SMALLINT,
|
||||
place_id BIGINT NOT NULL,
|
||||
partition SMALLINT NOT NULL,
|
||||
country_code VARCHAR(2),
|
||||
geometry GEOMETRY(Geometry, 4326)
|
||||
geometry GEOMETRY(Geometry, 4326) NOT NULL
|
||||
) {{db.tablespace.address_data}};
|
||||
CREATE INDEX idx_location_road_{{ partition }}_geometry ON location_road_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_location_road_{{ partition }}_place_id ON location_road_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
|
||||
|
||||
CREATE INDEX idx_location_road_{{ partition }}_geometry
|
||||
ON location_road_{{ partition }}
|
||||
USING GIST (geometry) {{db.tablespace.address_index}};
|
||||
CREATE UNIQUE INDEX idx_location_road_{{ partition }}_place_id
|
||||
ON location_road_{{ partition }}
|
||||
USING BTREE (place_id) {{db.tablespace.address_index}};
|
||||
{% endfor %}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2025 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- insert creates the location tables, creates location indexes if indexed == true
|
||||
@@ -31,3 +31,8 @@ CREATE TRIGGER location_postcodes_before_delete BEFORE DELETE ON location_postco
|
||||
FOR EACH ROW EXECUTE PROCEDURE postcodes_delete();
|
||||
CREATE TRIGGER location_postcodes_before_insert BEFORE INSERT ON location_postcodes
|
||||
FOR EACH ROW EXECUTE PROCEDURE postcodes_insert();
|
||||
|
||||
CREATE TRIGGER place_interpolation_before_insert BEFORE INSERT ON place_interpolation
|
||||
FOR EACH ROW EXECUTE PROCEDURE place_interpolation_insert();
|
||||
CREATE TRIGGER place_interpolation_before_delete BEFORE DELETE ON place_interpolation
|
||||
FOR EACH ROW EXECUTE PROCEDURE place_interpolation_delete();
|
||||
|
||||
@@ -2,312 +2,24 @@
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2025 by the Nominatim developer community.
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
drop table if exists import_status;
|
||||
CREATE TABLE import_status (
|
||||
lastimportdate timestamp with time zone NOT NULL,
|
||||
sequence_id integer,
|
||||
indexed boolean
|
||||
);
|
||||
GRANT SELECT ON import_status TO "{{config.DATABASE_WEBUSER}}" ;
|
||||
|
||||
drop table if exists import_osmosis_log;
|
||||
CREATE TABLE import_osmosis_log (
|
||||
batchend timestamp,
|
||||
batchseq integer,
|
||||
batchsize bigint,
|
||||
starttime timestamp,
|
||||
endtime timestamp,
|
||||
event text
|
||||
);
|
||||
|
||||
CREATE TABLE new_query_log (
|
||||
type text,
|
||||
starttime timestamp,
|
||||
ipaddress text,
|
||||
useragent text,
|
||||
language text,
|
||||
query text,
|
||||
searchterm text,
|
||||
endtime timestamp,
|
||||
results integer,
|
||||
format text,
|
||||
secret text
|
||||
);
|
||||
CREATE INDEX idx_new_query_log_starttime ON new_query_log USING BTREE (starttime);
|
||||
GRANT INSERT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
|
||||
GRANT UPDATE ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
|
||||
GRANT SELECT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
|
||||
|
||||
GRANT SELECT ON TABLE country_name TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
DROP TABLE IF EXISTS nominatim_properties;
|
||||
CREATE TABLE nominatim_properties (
|
||||
property TEXT NOT NULL,
|
||||
value TEXT
|
||||
);
|
||||
GRANT SELECT ON TABLE nominatim_properties TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
drop table IF EXISTS location_area CASCADE;
|
||||
CREATE TABLE location_area (
|
||||
place_id BIGINT,
|
||||
keywords INTEGER[],
|
||||
partition SMALLINT,
|
||||
rank_search SMALLINT NOT NULL,
|
||||
rank_address SMALLINT NOT NULL,
|
||||
country_code VARCHAR(2),
|
||||
isguess BOOL,
|
||||
postcode TEXT,
|
||||
centroid GEOMETRY(Point, 4326),
|
||||
geometry GEOMETRY(Geometry, 4326)
|
||||
);
|
||||
|
||||
CREATE TABLE location_area_large () INHERITS (location_area);
|
||||
|
||||
DROP TABLE IF EXISTS location_area_country;
|
||||
CREATE TABLE location_area_country (
|
||||
place_id BIGINT,
|
||||
country_code varchar(2),
|
||||
geometry GEOMETRY(Geometry, 4326)
|
||||
) {{db.tablespace.address_data}};
|
||||
CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {{db.tablespace.address_index}};
|
||||
|
||||
|
||||
CREATE TABLE location_property_tiger (
|
||||
place_id BIGINT,
|
||||
parent_place_id BIGINT,
|
||||
startnumber INTEGER,
|
||||
endnumber INTEGER,
|
||||
step SMALLINT,
|
||||
partition SMALLINT,
|
||||
linegeo GEOMETRY,
|
||||
postcode TEXT);
|
||||
GRANT SELECT ON location_property_tiger TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
drop table if exists location_property_osmline;
|
||||
CREATE TABLE location_property_osmline (
|
||||
place_id BIGINT NOT NULL,
|
||||
osm_id BIGINT,
|
||||
parent_place_id BIGINT,
|
||||
geometry_sector INTEGER,
|
||||
indexed_date TIMESTAMP,
|
||||
startnumber INTEGER,
|
||||
endnumber INTEGER,
|
||||
step SMALLINT,
|
||||
partition SMALLINT,
|
||||
indexed_status SMALLINT,
|
||||
linegeo GEOMETRY,
|
||||
address HSTORE,
|
||||
token_info JSONB, -- custom column for tokenizer use only
|
||||
postcode TEXT,
|
||||
country_code VARCHAR(2)
|
||||
){{db.tablespace.search_data}};
|
||||
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {{db.tablespace.search_index}}
|
||||
WHERE startnumber is not null;
|
||||
GRANT SELECT ON location_property_osmline TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
drop table IF EXISTS search_name;
|
||||
{% if not db.reverse_only %}
|
||||
CREATE TABLE search_name (
|
||||
place_id BIGINT,
|
||||
importance FLOAT,
|
||||
search_rank SMALLINT,
|
||||
address_rank SMALLINT,
|
||||
name_vector integer[],
|
||||
nameaddress_vector integer[],
|
||||
country_code varchar(2),
|
||||
centroid GEOMETRY(Geometry, 4326)
|
||||
) {{db.tablespace.search_data}};
|
||||
CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
GRANT SELECT ON search_name to "{{config.DATABASE_WEBUSER}}" ;
|
||||
{% endif %}
|
||||
|
||||
drop table IF EXISTS place_addressline;
|
||||
CREATE TABLE place_addressline (
|
||||
place_id BIGINT,
|
||||
address_place_id BIGINT,
|
||||
distance FLOAT,
|
||||
cached_rank_address SMALLINT,
|
||||
fromarea boolean,
|
||||
isaddress boolean
|
||||
) {{db.tablespace.search_data}};
|
||||
CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
|
||||
--------- PLACEX - storage for all indexed places -----------------
|
||||
|
||||
DROP TABLE IF EXISTS placex;
|
||||
CREATE TABLE placex (
|
||||
place_id BIGINT NOT NULL,
|
||||
parent_place_id BIGINT,
|
||||
linked_place_id BIGINT,
|
||||
importance FLOAT,
|
||||
indexed_date TIMESTAMP,
|
||||
geometry_sector INTEGER,
|
||||
rank_address SMALLINT,
|
||||
rank_search SMALLINT,
|
||||
partition SMALLINT,
|
||||
indexed_status SMALLINT,
|
||||
LIKE place INCLUDING CONSTRAINTS,
|
||||
wikipedia TEXT, -- calculated wikipedia article name (language:title)
|
||||
token_info JSONB, -- custom column for tokenizer use only
|
||||
country_code varchar(2),
|
||||
housenumber TEXT,
|
||||
postcode TEXT,
|
||||
centroid GEOMETRY(Geometry, 4326)
|
||||
) {{db.tablespace.search_data}};
|
||||
|
||||
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
{% for osm_type in ('N', 'W', 'R') %}
|
||||
CREATE INDEX idx_placex_osmid_{{osm_type | lower}} ON placex
|
||||
USING BTREE (osm_id) {{db.tablespace.search_index}}
|
||||
WHERE osm_type = '{{osm_type}}';
|
||||
{% endfor %}
|
||||
|
||||
-- Usage: - removing linkage status on update
|
||||
-- - lookup linked places for /details
|
||||
CREATE INDEX idx_placex_linked_place_id ON placex
|
||||
USING BTREE (linked_place_id) {{db.tablespace.address_index}}
|
||||
WHERE linked_place_id IS NOT NULL;
|
||||
|
||||
-- Usage: - check that admin boundaries do not overtake each other rank-wise
|
||||
-- - check that place node in a admin boundary with the same address level
|
||||
-- - boundary is not completely contained in a place area
|
||||
-- - parenting of large-area or unparentable features
|
||||
CREATE INDEX idx_placex_geometry_address_area_candidates ON placex
|
||||
USING gist (geometry) {{db.tablespace.address_index}}
|
||||
WHERE rank_address between 1 and 25
|
||||
and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
|
||||
|
||||
-- Usage: - POI is within building with housenumber
|
||||
CREATE INDEX idx_placex_geometry_buildings ON placex
|
||||
USING SPGIST (geometry) {{db.tablespace.address_index}}
|
||||
WHERE address is not null and rank_search = 30
|
||||
and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
|
||||
|
||||
-- Usage: - linking of similar named places to boundaries
|
||||
-- - linking of place nodes with same type to boundaries
|
||||
CREATE INDEX idx_placex_geometry_placenode ON placex
|
||||
USING SPGIST (geometry) {{db.tablespace.address_index}}
|
||||
WHERE osm_type = 'N' and rank_search < 26 and class = 'place';
|
||||
|
||||
-- Usage: - is node part of a way?
|
||||
-- - find parent of interpolation spatially
|
||||
CREATE INDEX idx_placex_geometry_lower_rank_ways ON placex
|
||||
USING SPGIST (geometry) {{db.tablespace.address_index}}
|
||||
WHERE osm_type = 'W' and rank_search >= 26;
|
||||
|
||||
-- Usage: - linking place nodes by wikidata tag to boundaries
|
||||
CREATE INDEX idx_placex_wikidata on placex
|
||||
USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}}
|
||||
WHERE extratags ? 'wikidata' and class = 'place'
|
||||
and osm_type = 'N' and rank_search < 26;
|
||||
|
||||
-- The following two indexes function as a todo list for indexing.
|
||||
|
||||
CREATE INDEX idx_placex_rank_address_sector ON placex
|
||||
USING BTREE (rank_address, geometry_sector) {{db.tablespace.address_index}}
|
||||
WHERE indexed_status > 0;
|
||||
|
||||
CREATE INDEX idx_placex_rank_boundaries_sector ON placex
|
||||
USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}}
|
||||
WHERE class = 'boundary' and type = 'administrative'
|
||||
and indexed_status > 0;
|
||||
|
||||
|
||||
DROP SEQUENCE IF EXISTS seq_place;
|
||||
CREATE SEQUENCE seq_place start 1;
|
||||
GRANT SELECT on placex to "{{config.DATABASE_WEBUSER}}" ;
|
||||
GRANT SELECT on place_addressline to "{{config.DATABASE_WEBUSER}}" ;
|
||||
GRANT SELECT ON planet_osm_ways to "{{config.DATABASE_WEBUSER}}" ;
|
||||
GRANT SELECT ON planet_osm_rels to "{{config.DATABASE_WEBUSER}}" ;
|
||||
GRANT SELECT on location_area to "{{config.DATABASE_WEBUSER}}" ;
|
||||
|
||||
-- Table for synthetic postcodes.
|
||||
DROP TABLE IF EXISTS location_postcodes;
|
||||
CREATE TABLE location_postcodes (
|
||||
place_id BIGINT NOT NULL,
|
||||
parent_place_id BIGINT,
|
||||
osm_id BIGINT,
|
||||
rank_search SMALLINT,
|
||||
indexed_status SMALLINT,
|
||||
indexed_date TIMESTAMP,
|
||||
country_code varchar(2),
|
||||
postcode TEXT NOT NULL,
|
||||
centroid GEOMETRY(Geometry, 4326) NOT NULL,
|
||||
geometry GEOMETRY(Geometry, 4326) NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_location_postcodes_id ON location_postcodes
|
||||
USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
CREATE INDEX idx_location_postcodes_geometry ON location_postcodes
|
||||
USING GIST (geometry) {{db.tablespace.search_index}};
|
||||
CREATE INDEX IF NOT EXISTS idx_location_postcodes_postcode
|
||||
ON location_postcodes USING BTREE (postcode, country_code)
|
||||
{{db.tablespace.search_index}};
|
||||
CREATE INDEX IF NOT EXISTS idx_location_postcodes_osmid
|
||||
ON location_postcodes USING BTREE (osm_id) {{db.tablespace.search_index}};
|
||||
GRANT SELECT ON location_postcodes TO "{{config.DATABASE_WEBUSER}}" ;
|
||||
|
||||
-- Table to store location of entrance nodes
|
||||
DROP TABLE IF EXISTS placex_entrance;
|
||||
CREATE TABLE placex_entrance (
|
||||
place_id BIGINT NOT NULL,
|
||||
osm_id BIGINT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
location GEOMETRY(Point, 4326) NOT NULL,
|
||||
extratags HSTORE
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_placex_entrance_place_id_osm_id ON placex_entrance
|
||||
USING BTREE (place_id, osm_id) {{db.tablespace.search_index}};
|
||||
GRANT SELECT ON placex_entrance TO "{{config.DATABASE_WEBUSER}}" ;
|
||||
|
||||
-- Create an index on the place table for lookups to populate the entrance
|
||||
-- table
|
||||
CREATE INDEX IF NOT EXISTS idx_placex_entrance_lookup ON place
|
||||
USING BTREE (osm_id)
|
||||
WHERE class IN ('routing:entrance', 'entrance');
|
||||
|
||||
DROP TABLE IF EXISTS import_polygon_error;
|
||||
CREATE TABLE import_polygon_error (
|
||||
osm_id BIGINT,
|
||||
osm_type CHAR(1),
|
||||
class TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
name HSTORE,
|
||||
country_code varchar(2),
|
||||
updated timestamp,
|
||||
errormessage text,
|
||||
prevgeometry GEOMETRY(Geometry, 4326),
|
||||
newgeometry GEOMETRY(Geometry, 4326)
|
||||
);
|
||||
CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
|
||||
GRANT SELECT ON import_polygon_error TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
DROP TABLE IF EXISTS import_polygon_delete;
|
||||
CREATE TABLE import_polygon_delete (
|
||||
osm_id BIGINT,
|
||||
osm_type CHAR(1),
|
||||
class TEXT NOT NULL,
|
||||
type TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id);
|
||||
GRANT SELECT ON import_polygon_delete TO "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
DROP SEQUENCE IF EXISTS file;
|
||||
CREATE SEQUENCE file start 1;
|
||||
|
||||
{% if 'wikimedia_importance' not in db.tables and 'wikipedia_article' not in db.tables %}
|
||||
-- create dummy tables here, if nothing was imported
|
||||
CREATE TABLE wikimedia_importance (
|
||||
language TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
importance double precision NOT NULL,
|
||||
wikidata TEXT
|
||||
) {{db.tablespace.address_data}};
|
||||
{% endif %}
|
||||
{% include('tables/status.sql') %}
|
||||
{% include('tables/nominatim_properties.sql') %}
|
||||
{% include('tables/location_area.sql') %}
|
||||
{% include('tables/tiger.sql') %}
|
||||
{% include('tables/interpolation.sql') %}
|
||||
{% include('tables/search_name.sql') %}
|
||||
{% include('tables/addressline.sql') %}
|
||||
{% include('tables/placex.sql') %}
|
||||
{% include('tables/postcodes.sql') %}
|
||||
{% include('tables/entrance.sql') %}
|
||||
{% include('tables/import_reports.sql') %}
|
||||
{% include('tables/importance_tables.sql') %}
|
||||
|
||||
-- osm2pgsql does not create indexes on the middle tables for Nominatim
|
||||
-- Add one for lookup of associated street relations.
|
||||
@@ -320,10 +32,3 @@ CREATE INDEX planet_osm_rels_relation_members_idx ON planet_osm_rels USING gin(p
|
||||
WITH (fastupdate=off)
|
||||
{{db.tablespace.address_index}};
|
||||
{% endif %}
|
||||
|
||||
-- Needed for lookups if a node is part of an interpolation.
|
||||
CREATE INDEX IF NOT EXISTS idx_place_interpolations
|
||||
ON place USING gist(geometry) {{db.tablespace.address_index}}
|
||||
WHERE osm_type = 'W' and address ? 'interpolation';
|
||||
|
||||
GRANT SELECT ON table country_osm_grid to "{{config.DATABASE_WEBUSER}}";
|
||||
|
||||
20
lib-sql/tables/addressline.sql
Normal file
20
lib-sql/tables/addressline.sql
Normal file
@@ -0,0 +1,20 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS place_addressline;
|
||||
|
||||
CREATE TABLE place_addressline (
|
||||
place_id BIGINT NOT NULL,
|
||||
address_place_id BIGINT NOT NULL,
|
||||
distance FLOAT NOT NULL,
|
||||
cached_rank_address SMALLINT NOT NULL,
|
||||
fromarea boolean NOT NULL,
|
||||
isaddress boolean NOT NULL
|
||||
) {{db.tablespace.search_data}};
|
||||
|
||||
CREATE INDEX idx_place_addressline_place_id ON place_addressline
|
||||
USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
20
lib-sql/tables/entrance.sql
Normal file
20
lib-sql/tables/entrance.sql
Normal file
@@ -0,0 +1,20 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- Table to store location of entrance nodes
|
||||
DROP TABLE IF EXISTS placex_entrance;
|
||||
|
||||
CREATE TABLE placex_entrance (
|
||||
place_id BIGINT NOT NULL,
|
||||
osm_id BIGINT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
location GEOMETRY(Point, 4326) NOT NULL,
|
||||
extratags HSTORE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_placex_entrance_place_id_osm_id ON placex_entrance
|
||||
USING BTREE (place_id, osm_id) {{db.tablespace.search_index}};
|
||||
35
lib-sql/tables/import_reports.sql
Normal file
35
lib-sql/tables/import_reports.sql
Normal file
@@ -0,0 +1,35 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS import_polygon_error;
|
||||
CREATE TABLE import_polygon_error (
|
||||
osm_id BIGINT,
|
||||
osm_type CHAR(1),
|
||||
class TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
name HSTORE,
|
||||
country_code varchar(2),
|
||||
updated timestamp,
|
||||
errormessage text,
|
||||
prevgeometry GEOMETRY(Geometry, 4326),
|
||||
newgeometry GEOMETRY(Geometry, 4326)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error
|
||||
USING BTREE (osm_type, osm_id);
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS import_polygon_delete;
|
||||
CREATE TABLE import_polygon_delete (
|
||||
osm_id BIGINT,
|
||||
osm_type CHAR(1),
|
||||
class TEXT NOT NULL,
|
||||
type TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete
|
||||
USING BTREE (osm_type, osm_id);
|
||||
16
lib-sql/tables/importance_tables.sql
Normal file
16
lib-sql/tables/importance_tables.sql
Normal file
@@ -0,0 +1,16 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
{% if 'wikimedia_importance' not in db.tables and 'wikipedia_article' not in db.tables %}
|
||||
-- create dummy tables here if nothing was imported
|
||||
CREATE TABLE wikimedia_importance (
|
||||
language TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
importance double precision NOT NULL,
|
||||
wikidata TEXT
|
||||
) {{db.tablespace.address_data}};
|
||||
{% endif %}
|
||||
35
lib-sql/tables/interpolation.sql
Normal file
35
lib-sql/tables/interpolation.sql
Normal file
@@ -0,0 +1,35 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS location_property_osmline;
|
||||
CREATE TABLE location_property_osmline (
|
||||
place_id BIGINT NOT NULL,
|
||||
osm_id BIGINT NOT NULL,
|
||||
parent_place_id BIGINT,
|
||||
geometry_sector INTEGER NOT NULL,
|
||||
indexed_date TIMESTAMP,
|
||||
type TEXT,
|
||||
startnumber INTEGER,
|
||||
endnumber INTEGER,
|
||||
step SMALLINT,
|
||||
partition SMALLINT NOT NULL,
|
||||
indexed_status SMALLINT NOT NULL,
|
||||
linegeo GEOMETRY(Geometry, 4326) NOT NULL,
|
||||
address HSTORE,
|
||||
token_info JSONB, -- custom column for tokenizer use only
|
||||
postcode TEXT,
|
||||
country_code VARCHAR(2)
|
||||
){{db.tablespace.search_data}};
|
||||
|
||||
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline
|
||||
USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline
|
||||
USING BTREE (geometry_sector) {{db.tablespace.address_index}};
|
||||
CREATE INDEX idx_osmline_linegeo ON location_property_osmline
|
||||
USING GIST (linegeo) {{db.tablespace.search_index}}
|
||||
WHERE startnumber is not null;
|
||||
|
||||
32
lib-sql/tables/location_area.sql
Normal file
32
lib-sql/tables/location_area.sql
Normal file
@@ -0,0 +1,32 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS location_area CASCADE;
|
||||
CREATE TABLE location_area (
|
||||
place_id BIGINT NOT NULL,
|
||||
keywords INTEGER[] NOT NULL,
|
||||
partition SMALLINT NOT NULL,
|
||||
rank_search SMALLINT NOT NULL,
|
||||
rank_address SMALLINT NOT NULL,
|
||||
country_code VARCHAR(2),
|
||||
isguess BOOL NOT NULL,
|
||||
postcode TEXT,
|
||||
centroid GEOMETRY(Point, 4326) NOT NULL,
|
||||
geometry GEOMETRY(Geometry, 4326) NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE location_area_large () INHERITS (location_area);
|
||||
|
||||
DROP TABLE IF EXISTS location_area_country;
|
||||
CREATE TABLE location_area_country (
|
||||
place_id BIGINT NOT NULL,
|
||||
country_code varchar(2) NOT NULL,
|
||||
geometry GEOMETRY(Geometry, 4326) NOT NULL
|
||||
) {{db.tablespace.address_data}};
|
||||
|
||||
CREATE INDEX idx_location_area_country_geometry ON location_area_country
|
||||
USING GIST (geometry) {{db.tablespace.address_index}};
|
||||
12
lib-sql/tables/nominatim_properties.sql
Normal file
12
lib-sql/tables/nominatim_properties.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS nominatim_properties;
|
||||
CREATE TABLE nominatim_properties (
|
||||
property TEXT NOT NULL,
|
||||
value TEXT
|
||||
);
|
||||
87
lib-sql/tables/placex.sql
Normal file
87
lib-sql/tables/placex.sql
Normal file
@@ -0,0 +1,87 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
-- placex - main table for searchable places
|
||||
|
||||
DROP TABLE IF EXISTS placex;
|
||||
CREATE TABLE placex (
|
||||
place_id BIGINT NOT NULL,
|
||||
parent_place_id BIGINT,
|
||||
linked_place_id BIGINT,
|
||||
importance FLOAT,
|
||||
indexed_date TIMESTAMP,
|
||||
geometry_sector INTEGER NOT NULL,
|
||||
rank_address SMALLINT NOT NULL,
|
||||
rank_search SMALLINT NOT NULL,
|
||||
partition SMALLINT NOT NULL,
|
||||
indexed_status SMALLINT NOT NULL,
|
||||
LIKE place INCLUDING CONSTRAINTS,
|
||||
wikipedia TEXT, -- calculated wikipedia article name (language:title)
|
||||
token_info JSONB, -- custom column for tokenizer use only
|
||||
country_code varchar(2),
|
||||
housenumber TEXT,
|
||||
postcode TEXT,
|
||||
centroid GEOMETRY(Geometry, 4326) NOT NULL
|
||||
) {{db.tablespace.search_data}};
|
||||
|
||||
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
{% for osm_type in ('N', 'W', 'R') %}
|
||||
CREATE INDEX idx_placex_osmid_{{osm_type | lower}} ON placex
|
||||
USING BTREE (osm_id) {{db.tablespace.search_index}}
|
||||
WHERE osm_type = '{{osm_type}}';
|
||||
{% endfor %}
|
||||
|
||||
-- Usage: - removing linkage status on update
|
||||
-- - lookup linked places for /details
|
||||
CREATE INDEX idx_placex_linked_place_id ON placex
|
||||
USING BTREE (linked_place_id) {{db.tablespace.address_index}}
|
||||
WHERE linked_place_id IS NOT NULL;
|
||||
|
||||
-- Usage: - check that admin boundaries do not overtake each other rank-wise
|
||||
-- - check that place node in a admin boundary with the same address level
|
||||
-- - boundary is not completely contained in a place area
|
||||
-- - parenting of large-area or unparentable features
|
||||
CREATE INDEX idx_placex_geometry_address_area_candidates ON placex
|
||||
USING gist (geometry) {{db.tablespace.address_index}}
|
||||
WHERE rank_address between 1 and 25
|
||||
and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
|
||||
|
||||
-- Usage: - POI is within building with housenumber
|
||||
CREATE INDEX idx_placex_geometry_buildings ON placex
|
||||
USING SPGIST (geometry) {{db.tablespace.address_index}}
|
||||
WHERE address is not null and rank_search = 30
|
||||
and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
|
||||
|
||||
-- Usage: - linking of similar named places to boundaries
|
||||
-- - linking of place nodes with same type to boundaries
|
||||
CREATE INDEX idx_placex_geometry_placenode ON placex
|
||||
USING SPGIST (geometry) {{db.tablespace.address_index}}
|
||||
WHERE osm_type = 'N' and rank_search < 26 and class = 'place';
|
||||
|
||||
-- Usage: - is node part of a way?
|
||||
-- - find parent of interpolation spatially
|
||||
CREATE INDEX idx_placex_geometry_lower_rank_ways ON placex
|
||||
USING SPGIST (geometry) {{db.tablespace.address_index}}
|
||||
WHERE osm_type = 'W' and rank_search >= 26;
|
||||
|
||||
-- Usage: - linking place nodes by wikidata tag to boundaries
|
||||
CREATE INDEX idx_placex_wikidata on placex
|
||||
USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}}
|
||||
WHERE extratags ? 'wikidata' and class = 'place'
|
||||
and osm_type = 'N' and rank_search < 26;
|
||||
|
||||
-- The following two indexes function as a todo list for indexing.
|
||||
|
||||
CREATE INDEX idx_placex_rank_address_sector ON placex
|
||||
USING BTREE (rank_address, geometry_sector) {{db.tablespace.address_index}}
|
||||
WHERE indexed_status > 0;
|
||||
|
||||
CREATE INDEX idx_placex_rank_boundaries_sector ON placex
|
||||
USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}}
|
||||
WHERE class = 'boundary' and type = 'administrative'
|
||||
and indexed_status > 0;
|
||||
|
||||
32
lib-sql/tables/postcodes.sql
Normal file
32
lib-sql/tables/postcodes.sql
Normal file
@@ -0,0 +1,32 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS location_postcodes;
|
||||
CREATE TABLE location_postcodes (
|
||||
place_id BIGINT NOT NULL,
|
||||
parent_place_id BIGINT,
|
||||
osm_id BIGINT,
|
||||
rank_search SMALLINT NOT NULL,
|
||||
indexed_status SMALLINT NOT NULL,
|
||||
indexed_date TIMESTAMP,
|
||||
country_code varchar(2) NOT NULL,
|
||||
postcode TEXT NOT NULL,
|
||||
centroid GEOMETRY(Geometry, 4326) NOT NULL,
|
||||
geometry GEOMETRY(Geometry, 4326) NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_location_postcodes_id ON location_postcodes
|
||||
USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
CREATE INDEX idx_location_postcodes_geometry ON location_postcodes
|
||||
USING GIST (geometry) {{db.tablespace.search_index}};
|
||||
CREATE INDEX idx_location_postcodes_centroid ON location_postcodes
|
||||
USING GIST (centroid) {{db.tablespace.search_index}};
|
||||
CREATE INDEX IF NOT EXISTS idx_location_postcodes_postcode ON location_postcodes
|
||||
USING BTREE (postcode, country_code) {{db.tablespace.search_index}};
|
||||
CREATE INDEX IF NOT EXISTS idx_location_postcodes_osmid ON location_postcodes
|
||||
USING BTREE (osm_id) {{db.tablespace.search_index}};
|
||||
|
||||
26
lib-sql/tables/search_name.sql
Normal file
26
lib-sql/tables/search_name.sql
Normal file
@@ -0,0 +1,26 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS search_name;
|
||||
|
||||
{% if not create_reverse_only %}
|
||||
|
||||
CREATE TABLE search_name (
|
||||
place_id BIGINT NOT NULL,
|
||||
importance FLOAT NOT NULL,
|
||||
search_rank SMALLINT NOT NULL,
|
||||
address_rank SMALLINT NOT NULL,
|
||||
name_vector integer[] NOT NULL,
|
||||
nameaddress_vector integer[] NOT NULL,
|
||||
country_code varchar(2),
|
||||
centroid GEOMETRY(Geometry, 4326) NOT NULL
|
||||
) {{db.tablespace.search_data}};
|
||||
|
||||
CREATE UNIQUE INDEX idx_search_name_place_id
|
||||
ON search_name USING BTREE (place_id) {{db.tablespace.search_index}};
|
||||
|
||||
{% endif %}
|
||||
23
lib-sql/tables/status.sql
Normal file
23
lib-sql/tables/status.sql
Normal file
@@ -0,0 +1,23 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS import_status;
|
||||
CREATE TABLE import_status (
|
||||
lastimportdate TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
sequence_id INTEGER,
|
||||
indexed BOOLEAN
|
||||
);
|
||||
|
||||
DROP TABLE IF EXISTS import_osmosis_log;
|
||||
CREATE TABLE import_osmosis_log (
|
||||
batchend TIMESTAMP,
|
||||
batchseq INTEGER,
|
||||
batchsize BIGINT,
|
||||
starttime TIMESTAMP,
|
||||
endtime TIMESTAMP,
|
||||
event TEXT
|
||||
);
|
||||
17
lib-sql/tables/tiger.sql
Normal file
17
lib-sql/tables/tiger.sql
Normal file
@@ -0,0 +1,17 @@
|
||||
-- SPDX-License-Identifier: GPL-2.0-only
|
||||
--
|
||||
-- This file is part of Nominatim. (https://nominatim.org)
|
||||
--
|
||||
-- Copyright (C) 2026 by the Nominatim developer community.
|
||||
-- For a full list of authors see the git log.
|
||||
|
||||
DROP TABLE IF EXISTS location_property_tiger;
|
||||
CREATE TABLE location_property_tiger (
|
||||
place_id BIGINT NOT NULL,
|
||||
parent_place_id BIGINT,
|
||||
startnumber INTEGER NOT NULL,
|
||||
endnumber INTEGER NOT NULL,
|
||||
step SMALLINT NOT NULL,
|
||||
partition SMALLINT NOT NULL,
|
||||
linegeo GEOMETRY NOT NULL,
|
||||
postcode TEXT);
|
||||
@@ -15,6 +15,99 @@ CREATE TABLE location_property_tiger_import (
|
||||
step SMALLINT,
|
||||
postcode TEXT);
|
||||
|
||||
|
||||
-- Lookup functions for tiger import when update
|
||||
-- tables are dropped (see gh-issue #2463)
|
||||
CREATE OR REPLACE FUNCTION getNearestNamedRoadPlaceIdSlow(in_centroid GEOMETRY,
|
||||
in_token_info JSONB)
|
||||
RETURNS BIGINT
|
||||
AS $$
|
||||
DECLARE
|
||||
out_place_id BIGINT;
|
||||
|
||||
BEGIN
|
||||
SELECT place_id INTO out_place_id
|
||||
FROM search_name
|
||||
WHERE
|
||||
-- finds rows where name_vector shares elements with search tokens.
|
||||
token_matches_street(in_token_info, name_vector)
|
||||
-- limits search area
|
||||
AND centroid && ST_Expand(in_centroid, 0.015)
|
||||
AND address_rank BETWEEN 26 AND 27
|
||||
ORDER BY ST_Distance(centroid, in_centroid) ASC
|
||||
LIMIT 1;
|
||||
|
||||
RETURN out_place_id;
|
||||
END
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION getNearestParallelRoadFeatureSlow(line GEOMETRY)
|
||||
RETURNS BIGINT
|
||||
AS $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
search_diameter FLOAT;
|
||||
p1 GEOMETRY;
|
||||
p2 GEOMETRY;
|
||||
p3 GEOMETRY;
|
||||
|
||||
BEGIN
|
||||
IF ST_GeometryType(line) not in ('ST_LineString') THEN
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
p1 := ST_LineInterpolatePoint(line,0);
|
||||
p2 := ST_LineInterpolatePoint(line,0.5);
|
||||
p3 := ST_LineInterpolatePoint(line,1);
|
||||
|
||||
search_diameter := 0.0005;
|
||||
WHILE search_diameter < 0.01 LOOP
|
||||
FOR r IN
|
||||
SELECT place_id FROM placex
|
||||
WHERE ST_DWithin(line, geometry, search_diameter)
|
||||
AND rank_address BETWEEN 26 AND 27
|
||||
ORDER BY (ST_distance(geometry, p1)+
|
||||
ST_distance(geometry, p2)+
|
||||
ST_distance(geometry, p3)) ASC limit 1
|
||||
LOOP
|
||||
RETURN r.place_id;
|
||||
END LOOP;
|
||||
search_diameter := search_diameter * 2;
|
||||
END LOOP;
|
||||
RETURN NULL;
|
||||
END
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION getNearestRoadPlaceIdSlow(point GEOMETRY)
|
||||
RETURNS BIGINT
|
||||
AS $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
search_diameter FLOAT;
|
||||
BEGIN
|
||||
search_diameter := 0.00005;
|
||||
WHILE search_diameter < 0.1 LOOP
|
||||
FOR r IN
|
||||
SELECT place_id FROM placex
|
||||
WHERE ST_DWithin(geometry, point, search_diameter)
|
||||
AND rank_address BETWEEN 26 AND 27
|
||||
ORDER BY ST_Distance(geometry, point) ASC limit 1
|
||||
LOOP
|
||||
RETURN r.place_id;
|
||||
END LOOP;
|
||||
search_diameter := search_diameter * 2;
|
||||
END LOOP;
|
||||
RETURN NULL;
|
||||
END
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
|
||||
-- Tiger import function
|
||||
CREATE OR REPLACE FUNCTION tiger_line_import(linegeo GEOMETRY, in_startnumber INTEGER,
|
||||
in_endnumber INTEGER, interpolationtype TEXT,
|
||||
token_info JSONB, in_postcode TEXT) RETURNS INTEGER
|
||||
@@ -71,28 +164,51 @@ BEGIN
|
||||
place_centroid := ST_Centroid(linegeo);
|
||||
out_partition := get_partition('us');
|
||||
|
||||
out_parent_place_id := getNearestNamedRoadPlaceId(out_partition, place_centroid,
|
||||
-- HYBRID LOOKUP LOGIC (see gh-issue #2463)
|
||||
-- if partition tables exist, use them for fast spatial lookups
|
||||
{% if 'location_road_0' in db.tables %}
|
||||
out_parent_place_id := getNearestNamedRoadPlaceId(out_partition, place_centroid,
|
||||
token_info);
|
||||
|
||||
IF out_parent_place_id IS NULL THEN
|
||||
SELECT getNearestParallelRoadFeature(out_partition, linegeo)
|
||||
INTO out_parent_place_id;
|
||||
IF out_parent_place_id IS NULL THEN
|
||||
SELECT getNearestParallelRoadFeature(out_partition, linegeo)
|
||||
INTO out_parent_place_id;
|
||||
END IF;
|
||||
|
||||
IF out_parent_place_id IS NULL THEN
|
||||
SELECT getNearestRoadPlaceId(out_partition, place_centroid)
|
||||
INTO out_parent_place_id;
|
||||
END IF;
|
||||
|
||||
-- When updatable information has been dropped:
|
||||
-- Partition tables no longer exist, but search_name still persists.
|
||||
{% elif 'search_name' in db.tables %}
|
||||
-- Fallback: Look up in 'search_name' table
|
||||
-- though spatial lookups here can be slower.
|
||||
out_parent_place_id := getNearestNamedRoadPlaceIdSlow(place_centroid, token_info);
|
||||
|
||||
IF out_parent_place_id IS NULL THEN
|
||||
out_parent_place_id := getNearestParallelRoadFeatureSlow(linegeo);
|
||||
END IF;
|
||||
|
||||
IF out_parent_place_id IS NULL THEN
|
||||
out_parent_place_id := getNearestRoadPlaceIdSlow(place_centroid);
|
||||
END IF;
|
||||
{% endif %}
|
||||
|
||||
-- If parent was found, insert street(line) into import table
|
||||
IF out_parent_place_id IS NOT NULL THEN
|
||||
INSERT INTO location_property_tiger_import (linegeo, place_id, partition,
|
||||
parent_place_id, startnumber, endnumber,
|
||||
step, postcode)
|
||||
VALUES (linegeo, nextval('seq_place'), out_partition,
|
||||
out_parent_place_id, startnumber, endnumber,
|
||||
stepsize, in_postcode);
|
||||
|
||||
RETURN 1;
|
||||
END IF;
|
||||
RETURN 0;
|
||||
|
||||
IF out_parent_place_id IS NULL THEN
|
||||
SELECT getNearestRoadPlaceId(out_partition, place_centroid)
|
||||
INTO out_parent_place_id;
|
||||
END IF;
|
||||
|
||||
--insert street(line) into import table
|
||||
insert into location_property_tiger_import (linegeo, place_id, partition,
|
||||
parent_place_id, startnumber, endnumber,
|
||||
step, postcode)
|
||||
values (linegeo, nextval('seq_place'), out_partition,
|
||||
out_parent_place_id, startnumber, endnumber,
|
||||
stepsize, in_postcode);
|
||||
|
||||
RETURN 1;
|
||||
END;
|
||||
$$
|
||||
LANGUAGE plpgsql;
|
||||
|
||||
@@ -23,7 +23,7 @@ an ASGI-capable server like uvicorn. To install them from pypi run:
|
||||
You need to have a Nominatim database imported with the 'nominatim-db'
|
||||
package. Go to the project directory, then run uvicorn as:
|
||||
|
||||
uvicorn --factory nominatim.server.falcon.server:run_wsgi
|
||||
uvicorn --factory nominatim_api.server.falcon.server:run_wsgi
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ class FormatDispatcher:
|
||||
return decorator
|
||||
|
||||
def error_format_func(self, func: ErrorFormatFunc) -> ErrorFormatFunc:
|
||||
""" Decorator for a function that formats error messges.
|
||||
""" Decorator for a function that formats error messages.
|
||||
There is only one error formatter per dispatcher. Using
|
||||
the decorator repeatedly will overwrite previous functions.
|
||||
"""
|
||||
@@ -79,7 +79,7 @@ class FormatDispatcher:
|
||||
def set_content_type(self, fmt: str, content_type: str) -> None:
|
||||
""" Set the content type for the given format. This is the string
|
||||
that will be returned in the Content-Type header of the HTML
|
||||
response, when the given format is choosen.
|
||||
response, when the given format is chosen.
|
||||
"""
|
||||
self.content_types[fmt] = content_type
|
||||
|
||||
|
||||
@@ -177,6 +177,7 @@ class SearchBuilder:
|
||||
sdata.lookups = partials.split_lookup(split, 'nameaddress_vector')
|
||||
sdata.lookups.append(
|
||||
dbf.FieldLookup('name_vector', hnr_tokens, lookups.Restrict))
|
||||
expected_count = partials.min_count() / (5**(split - 1))
|
||||
else:
|
||||
addr_fulls = [t.token for t in
|
||||
self.query.get_tokens(address[0], qmod.TOKEN_WORD)]
|
||||
|
||||
@@ -22,7 +22,7 @@ class CountedTokenIDs:
|
||||
""" A list of token IDs with their respective counts, sorted
|
||||
from least frequent to most frequent.
|
||||
|
||||
If a token count is one, then statistics are likely to be unavaible
|
||||
If a token count is one, then statistics are likely to be unavailable
|
||||
and a relatively high count is assumed instead.
|
||||
"""
|
||||
|
||||
|
||||
@@ -170,11 +170,20 @@ class ForwardGeocoder:
|
||||
if qword not in words:
|
||||
wdist = max(difflib.SequenceMatcher(a=qword, b=w).quick_ratio() for w in words)
|
||||
distance += len(qword) if wdist < 0.4 else 1
|
||||
# Compensate for the fact that country names do not get a
|
||||
# match penalty yet by the tokenizer.
|
||||
# Temporary hack that needs to be removed!
|
||||
# Countries with high importance can dominate results when matched
|
||||
# via an alternate-language name. Apply a language-aware penalty
|
||||
# to offset this.
|
||||
if result.rank_address == 4:
|
||||
distance *= 2
|
||||
if self.params.locales and result.names:
|
||||
loc_names = [result.names[t] for t in self.params.locales.name_tags
|
||||
if t in result.names]
|
||||
if loc_names:
|
||||
norm_loc = self.query_analyzer.normalize_text(' '.join(loc_names))
|
||||
loc_words = set(w for w in re.split('[-,: ]+', norm_loc) if w)
|
||||
if loc_words and loc_words.isdisjoint(qwords):
|
||||
result.accuracy += result.calculated_importance() * 0.5
|
||||
else:
|
||||
distance *= 2
|
||||
result.accuracy += distance * 0.3 / sum(len(w) for w in qwords)
|
||||
|
||||
async def lookup_pois(self, categories: List[Tuple[str, str]],
|
||||
|
||||
@@ -17,7 +17,7 @@ import dataclasses
|
||||
# The x value for the regression computation will be the position of the
|
||||
# token in the query. Thus we know the x values will be [0, query length).
|
||||
# As the denominator only depends on the x values, we can pre-compute here
|
||||
# the denominatior to use for a given query length.
|
||||
# the denominator to use for a given query length.
|
||||
# Note that query length of two or less is special cased and will not use
|
||||
# the values from this array. Thus it is not a problem that they are 0.
|
||||
LINFAC = [i * (sum(si * si for si in range(i)) - (i - 1) * i * (i - 1) / 4)
|
||||
@@ -129,7 +129,7 @@ class Token(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get_country(self) -> str:
|
||||
""" Return the country code this tojen is associated with
|
||||
""" Return the country code this token is associated with
|
||||
(currently for country tokens only).
|
||||
"""
|
||||
|
||||
@@ -231,7 +231,7 @@ class QueryNode:
|
||||
return max(0, -self.penalty)
|
||||
|
||||
def name_address_ratio(self) -> float:
|
||||
""" Return the propability that the partial token belonging to
|
||||
""" Return the probability that the partial token belonging to
|
||||
this node forms part of a name (as opposed of part of the address).
|
||||
"""
|
||||
if self.partial is None:
|
||||
@@ -275,7 +275,7 @@ class QueryStruct:
|
||||
directed acyclic graph.
|
||||
|
||||
A query also has a direction penalty 'dir_penalty'. This describes
|
||||
the likelyhood if the query should be read from left-to-right or
|
||||
the likelihood if the query should be read from left-to-right or
|
||||
vice versa. A negative 'dir_penalty' should be read as a penalty on
|
||||
right-to-left reading, while a positive value represents a penalty
|
||||
for left-to-right reading. The default value is 0, which is equivalent
|
||||
|
||||
@@ -184,6 +184,10 @@ class APIMiddleware:
|
||||
formatter = load_format_dispatcher('v1', self.api.config.project_dir)
|
||||
for name, func in await api_impl.get_routes(self.api):
|
||||
endpoint = EndpointWrapper(name, func, self.api, formatter)
|
||||
# If func is a LazySearchEndpoint, give it a reference to wrapper
|
||||
# so it can replace wrapper.func dynamically
|
||||
if hasattr(func, 'set_wrapper'):
|
||||
func.set_wrapper(endpoint)
|
||||
self.app.add_route(f"/{name}", endpoint)
|
||||
if legacy_urls:
|
||||
self.app.add_route(f"/{name}.php", endpoint)
|
||||
|
||||
@@ -50,7 +50,7 @@ class ParamWrapper(ASGIAdaptor):
|
||||
headers={'content-type': self.content_type})
|
||||
|
||||
def create_response(self, status: int, output: str, num_results: int) -> Response:
|
||||
self.request.state.num_results = num_results
|
||||
setattr(self.request.state, 'num_results', num_results)
|
||||
return Response(output, status_code=status, media_type=self.content_type)
|
||||
|
||||
def base_uri(self) -> str:
|
||||
@@ -95,7 +95,7 @@ class FileLoggingMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request,
|
||||
call_next: RequestResponseEndpoint) -> Response:
|
||||
qs = QueryStatistics()
|
||||
request.state.query_stats = qs
|
||||
setattr(request.state, 'query_stats', qs)
|
||||
response = await call_next(request)
|
||||
|
||||
if response.status_code != 200 or 'start' not in qs:
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
Complex datatypes used by the Nominatim API.
|
||||
"""
|
||||
from typing import Optional, Union, Tuple, NamedTuple, TypeVar, Type, Dict, \
|
||||
Any, List, Sequence
|
||||
Any, List, Sequence, TYPE_CHECKING
|
||||
from collections import abc
|
||||
import dataclasses
|
||||
import datetime as dt
|
||||
@@ -17,6 +17,8 @@ import math
|
||||
from struct import unpack
|
||||
from binascii import unhexlify
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .localization import Locales
|
||||
from .errors import UsageError
|
||||
|
||||
|
||||
@@ -573,6 +575,13 @@ class SearchDetails(LookupDetails):
|
||||
|
||||
viewbox_x2: Optional[Bbox] = None
|
||||
|
||||
locales: Optional['Locales'] = dataclasses.field(
|
||||
default=None, metadata={'transform': lambda v: v})
|
||||
""" Locale preferences of the caller.
|
||||
Used during result re-ranking to prefer results that match the
|
||||
caller's locale over results that only match in an alternate language.
|
||||
"""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.viewbox is not None:
|
||||
xext = (self.viewbox.maxlon - self.viewbox.minlon)/2
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Generic part of the server implementation of the v1 API.
|
||||
@@ -12,6 +12,7 @@ from typing import Optional, Any, Type, Dict, cast, Sequence, Tuple
|
||||
from functools import reduce
|
||||
import dataclasses
|
||||
from urllib.parse import urlencode
|
||||
import asyncio
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
@@ -124,6 +125,12 @@ def parse_geometry_details(adaptor: ASGIAdaptor, fmt: str) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def has_search_name(conn: sa.engine.Connection) -> bool:
|
||||
""" Check if the search_name table exists in the database.
|
||||
"""
|
||||
return sa.inspect(conn).has_table('search_name')
|
||||
|
||||
|
||||
async def status_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
""" Server glue for /status endpoint. See API docs for details.
|
||||
"""
|
||||
@@ -200,6 +207,7 @@ async def reverse_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
details['max_rank'] = helpers.zoom_to_rank(params.get_int('zoom', 18))
|
||||
details['layers'] = get_layers(params)
|
||||
details['query_stats'] = params.query_stats()
|
||||
details['entrances'] = params.get_bool('entrances', False)
|
||||
|
||||
result = await api.reverse(coord, **details)
|
||||
|
||||
@@ -238,6 +246,7 @@ async def lookup_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
debug = setup_debugging(params)
|
||||
details = parse_geometry_details(params, fmt)
|
||||
details['query_stats'] = params.query_stats()
|
||||
details['entrances'] = params.get_bool('entrances', False)
|
||||
|
||||
places = []
|
||||
for oid in (params.get('osm_ids') or '').split(','):
|
||||
@@ -325,6 +334,8 @@ async def search_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
details['layers'] = DataLayer.ADDRESS
|
||||
else:
|
||||
details['layers'] = get_layers(params)
|
||||
details['locales'] = Locales.from_accept_languages(get_accepted_languages(params),
|
||||
params.config().OUTPUT_NAMES)
|
||||
|
||||
# unstructured query parameters
|
||||
query = params.get('q', None)
|
||||
@@ -350,8 +361,7 @@ async def search_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
except UsageError as err:
|
||||
params.raise_error(str(err))
|
||||
|
||||
Locales.from_accept_languages(get_accepted_languages(params),
|
||||
params.config().OUTPUT_NAMES).localize_results(results)
|
||||
details['locales'].localize_results(results)
|
||||
|
||||
if details['dedupe'] and len(results) > 1:
|
||||
results = helpers.deduplicate_results(results, max_results)
|
||||
@@ -439,6 +449,61 @@ async def polygons_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
return build_response(params, params.formatting().format_result(results, fmt, {}))
|
||||
|
||||
|
||||
async def search_unavailable_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
""" Server glue for /search endpoint in reverse-only mode.
|
||||
Returns 404 when search functionality is not available.
|
||||
"""
|
||||
params.raise_error('Search not available (reverse-only mode)', 404)
|
||||
|
||||
|
||||
class LazySearchEndpoint:
|
||||
"""
|
||||
Lazy-loading search endpoint that replaces itself after first successful check.
|
||||
|
||||
- Falcon: EndpointWrapper stores this instance in wrapper.func
|
||||
On first request, replace wrapper.func directly with real endpoint
|
||||
|
||||
- Starlette: _wrap_endpoint wraps this instance in a callback
|
||||
store a delegate function and call it on subsequent requests
|
||||
"""
|
||||
def __init__(self, api: NominatimAPIAsync, real_endpoint: EndpointFunc):
|
||||
self.api = api
|
||||
self.real_endpoint = real_endpoint
|
||||
self._lock = asyncio.Lock()
|
||||
self._wrapper: Any = None # Store reference to Falcon's EndpointWrapper
|
||||
self._delegate: Optional[EndpointFunc] = None
|
||||
|
||||
def set_wrapper(self, wrapper: Any) -> None:
|
||||
self._wrapper = wrapper
|
||||
|
||||
async def __call__(self, api: NominatimAPIAsync, params: ASGIAdaptor) -> Any:
|
||||
if self._delegate is None:
|
||||
async with self._lock:
|
||||
# Double-check after acquiring lock (thread safety)
|
||||
if self._delegate is None:
|
||||
try:
|
||||
async with api.begin() as conn:
|
||||
has_table = await conn.connection.run_sync(
|
||||
has_search_name)
|
||||
|
||||
if has_table:
|
||||
# For Starlette
|
||||
self._delegate = self.real_endpoint
|
||||
# For Falcon
|
||||
if self._wrapper is not None:
|
||||
self._wrapper.func = self.real_endpoint
|
||||
else:
|
||||
self._delegate = search_unavailable_endpoint
|
||||
if self._wrapper is not None:
|
||||
self._wrapper.func = search_unavailable_endpoint
|
||||
|
||||
except (PGCORE_ERROR, sa.exc.OperationalError, OSError):
|
||||
# No _delegate set, so retry on next request
|
||||
params.raise_error('Search temporarily unavailable', 503)
|
||||
|
||||
return await self._delegate(api, params)
|
||||
|
||||
|
||||
async def get_routes(api: NominatimAPIAsync) -> Sequence[Tuple[str, EndpointFunc]]:
|
||||
routes = [
|
||||
('status', status_endpoint),
|
||||
@@ -449,15 +514,13 @@ async def get_routes(api: NominatimAPIAsync) -> Sequence[Tuple[str, EndpointFunc
|
||||
('polygons', polygons_endpoint),
|
||||
]
|
||||
|
||||
def has_search_name(conn: sa.engine.Connection) -> bool:
|
||||
insp = sa.inspect(conn)
|
||||
return insp.has_table('search_name')
|
||||
|
||||
try:
|
||||
async with api.begin() as conn:
|
||||
if await conn.connection.run_sync(has_search_name):
|
||||
routes.append(('search', search_endpoint))
|
||||
except (PGCORE_ERROR, sa.exc.OperationalError):
|
||||
pass # ignored
|
||||
else:
|
||||
routes.append(('search', search_unavailable_endpoint))
|
||||
except (PGCORE_ERROR, sa.exc.OperationalError, OSError):
|
||||
routes.append(('search', LazySearchEndpoint(api, search_endpoint)))
|
||||
|
||||
return routes
|
||||
|
||||
@@ -65,14 +65,14 @@ class UpdateAddData:
|
||||
def run(self, args: NominatimArgs) -> int:
|
||||
from ..tools import add_osm_data
|
||||
|
||||
if args.tiger_data:
|
||||
return asyncio.run(self._add_tiger_data(args))
|
||||
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
if is_frozen(conn):
|
||||
print('Database is marked frozen. New data can\'t be added.')
|
||||
return 1
|
||||
|
||||
if args.tiger_data:
|
||||
return asyncio.run(self._add_tiger_data(args))
|
||||
|
||||
osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
|
||||
if args.file or args.diff:
|
||||
return add_osm_data.add_data_from_file(args.config.get_libpq_dsn(),
|
||||
|
||||
@@ -104,7 +104,7 @@ def _get_locales(args: NominatimArgs, config: Configuration) -> napi.Locales:
|
||||
return napi.Locales()
|
||||
|
||||
|
||||
def _get_layers(args: NominatimArgs, default: napi.DataLayer) -> Optional[napi.DataLayer]:
|
||||
def _get_layers(args: NominatimArgs, default: Optional[napi.DataLayer]) -> Optional[napi.DataLayer]:
|
||||
""" Get the list of selected layers as a DataLayer enum.
|
||||
"""
|
||||
if not args.layers:
|
||||
@@ -136,7 +136,7 @@ def _print_output(formatter: napi.FormatDispatcher, result: Any,
|
||||
json.dump(json.loads(output), sys.stdout, indent=4, ensure_ascii=False)
|
||||
except json.decoder.JSONDecodeError as err:
|
||||
# Catch the error here, so that data can be debugged,
|
||||
# when people are developping custom result formatters.
|
||||
# when people are developing custom result formatters.
|
||||
LOG.fatal("Parsing json failed: %s\nUnformatted output:\n%s", err, output)
|
||||
else:
|
||||
sys.stdout.write(output)
|
||||
@@ -173,6 +173,10 @@ class APISearch:
|
||||
help='Preferred area to find search results')
|
||||
group.add_argument('--bounded', action='store_true',
|
||||
help='Strictly restrict results to viewbox area')
|
||||
group.add_argument('--layer', metavar='LAYER',
|
||||
choices=[n.name.lower() for n in napi.DataLayer if n.name],
|
||||
action='append', required=False, dest='layers',
|
||||
help='Restrict results to one or more layers (may be repeated)')
|
||||
group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
|
||||
help='Do not remove duplicates from the result list')
|
||||
_add_list_format(parser)
|
||||
@@ -189,6 +193,8 @@ class APISearch:
|
||||
raise UsageError(f"Unsupported format '{args.format}'. "
|
||||
'Use --list-formats to see supported formats.')
|
||||
|
||||
layers = _get_layers(args, None)
|
||||
|
||||
try:
|
||||
with napi.NominatimAPI(args.project_dir) as api:
|
||||
params: Dict[str, Any] = {'max_results': args.limit + min(args.limit, 10),
|
||||
@@ -199,6 +205,7 @@ class APISearch:
|
||||
'excluded': args.exclude_place_ids,
|
||||
'viewbox': args.viewbox,
|
||||
'bounded_viewbox': args.bounded,
|
||||
'layers': layers,
|
||||
'entrances': args.entrances,
|
||||
}
|
||||
|
||||
@@ -255,7 +262,7 @@ class APIReverse:
|
||||
group.add_argument('--layer', metavar='LAYER',
|
||||
choices=[n.name.lower() for n in napi.DataLayer if n.name],
|
||||
action='append', required=False, dest='layers',
|
||||
help='OSM id to lookup in format <NRW><id> (may be repeated)')
|
||||
help='Restrict results to one or more layers (may be repeated)')
|
||||
|
||||
_add_api_output_arguments(parser)
|
||||
_add_list_format(parser)
|
||||
|
||||
@@ -119,6 +119,8 @@ class NominatimArgs:
|
||||
enable_debug_statements: bool
|
||||
data_object: Sequence[Tuple[str, int]]
|
||||
data_area: Sequence[Tuple[str, int]]
|
||||
ro_access: bool
|
||||
postcode_force_reimport: bool
|
||||
|
||||
# Arguments to 'replication'
|
||||
init: bool
|
||||
|
||||
@@ -64,4 +64,4 @@ class UpdateIndex:
|
||||
if not args.boundaries_only:
|
||||
await indexer.index_by_rank(args.minrank, args.maxrank)
|
||||
await indexer.index_postcodes()
|
||||
has_pending = indexer.has_pending()
|
||||
has_pending = indexer.has_pending(args.minrank, args.maxrank)
|
||||
|
||||
@@ -65,6 +65,8 @@ class UpdateRefresh:
|
||||
help='Update secondary importance raster data')
|
||||
group.add_argument('--importance', action='store_true',
|
||||
help='Recompute place importances (expensive!)')
|
||||
group.add_argument('--ro-access', action='store_true',
|
||||
help='Grant read-only access to web user for all tables')
|
||||
group.add_argument('--website', action='store_true',
|
||||
help='DEPRECATED. This function has no function anymore'
|
||||
' and will be removed in a future version.')
|
||||
@@ -82,6 +84,10 @@ class UpdateRefresh:
|
||||
help='Do not enable code for propagating updates')
|
||||
group.add_argument('--enable-debug-statements', action='store_true',
|
||||
help='Enable debug warning statements in functions')
|
||||
group = parser.add_argument_group('Arguments for postcode refresh')
|
||||
group.add_argument('--force-reimport', action='store_true',
|
||||
dest='postcode_force_reimport',
|
||||
help='Recompute the postcodes from scratch instead of updating')
|
||||
|
||||
def run(self, args: NominatimArgs) -> int:
|
||||
from ..tools import refresh, postcodes
|
||||
@@ -94,7 +100,8 @@ class UpdateRefresh:
|
||||
LOG.warning("Update postcodes centroid")
|
||||
tokenizer = self._get_tokenizer(args.config)
|
||||
postcodes.update_postcodes(args.config.get_libpq_dsn(),
|
||||
args.project_dir, tokenizer)
|
||||
args.project_dir, tokenizer,
|
||||
force_reimport=args.postcode_force_reimport)
|
||||
indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
|
||||
args.threads or 1)
|
||||
asyncio.run(indexer.index_postcodes())
|
||||
@@ -159,6 +166,11 @@ class UpdateRefresh:
|
||||
LOG.error('WARNING: Website setup is no longer required. '
|
||||
'This function will be removed in future version of Nominatim.')
|
||||
|
||||
if args.ro_access:
|
||||
from ..tools import admin
|
||||
LOG.warning('Grant read-only access to web user')
|
||||
admin.grant_ro_access(args.config.get_libpq_dsn(), args.config)
|
||||
|
||||
if args.data_object or args.data_area:
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
for obj in args.data_object or []:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Nominatim configuration accessor.
|
||||
@@ -12,6 +12,7 @@ import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
import json
|
||||
import yaml
|
||||
@@ -80,6 +81,10 @@ class Configuration:
|
||||
self.lib_dir = _LibDirs()
|
||||
self._private_plugins: Dict[str, object] = {}
|
||||
|
||||
if re.fullmatch(r'[\w-]+', self.DATABASE_WEBUSER) is None:
|
||||
raise UsageError("Misconfigured DATABASE_WEBUSER. "
|
||||
"Only alphnumberic characters, - and _ are allowed.")
|
||||
|
||||
def set_libdirs(self, **kwargs: StrPath) -> None:
|
||||
""" Set paths to library functions and data.
|
||||
"""
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2024 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
A connection pool that executes incoming queries in parallel.
|
||||
@@ -27,20 +27,30 @@ class QueryPool:
|
||||
The results of the queries is discarded.
|
||||
"""
|
||||
def __init__(self, dsn: str, pool_size: int = 1, **conn_args: Any) -> None:
|
||||
self.is_cancelled = False
|
||||
self.wait_time = 0.0
|
||||
self.query_queue: 'asyncio.Queue[QueueItem]' = asyncio.Queue(maxsize=2 * pool_size)
|
||||
|
||||
self.pool = [asyncio.create_task(self._worker_loop(dsn, **conn_args))
|
||||
self.pool = [asyncio.create_task(self._worker_loop_cancellable(dsn, **conn_args))
|
||||
for _ in range(pool_size)]
|
||||
|
||||
async def put_query(self, query: psycopg.abc.Query, params: Any) -> None:
|
||||
""" Schedule a query for execution.
|
||||
"""
|
||||
if self.is_cancelled:
|
||||
self.clear_queue()
|
||||
await self.finish()
|
||||
return
|
||||
|
||||
tstart = time.time()
|
||||
await self.query_queue.put((query, params))
|
||||
self.wait_time += time.time() - tstart
|
||||
await asyncio.sleep(0)
|
||||
|
||||
if self.is_cancelled:
|
||||
self.clear_queue()
|
||||
await self.finish()
|
||||
|
||||
async def finish(self) -> None:
|
||||
""" Wait for all queries to finish and close the pool.
|
||||
"""
|
||||
@@ -56,6 +66,25 @@ class QueryPool:
|
||||
if excp is not None:
|
||||
raise excp
|
||||
|
||||
def clear_queue(self) -> None:
|
||||
""" Drop all items silently that might still be queued.
|
||||
"""
|
||||
try:
|
||||
while True:
|
||||
self.query_queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
pass # expected
|
||||
|
||||
async def _worker_loop_cancellable(self, dsn: str, **conn_args: Any) -> None:
|
||||
try:
|
||||
await self._worker_loop(dsn, **conn_args)
|
||||
except Exception as e:
|
||||
# Make sure the exception is forwarded to the main function
|
||||
self.is_cancelled = True
|
||||
# clear the queue here to ensure that any put() that may be blocked returns
|
||||
self.clear_queue()
|
||||
raise e
|
||||
|
||||
async def _worker_loop(self, dsn: str, **conn_args: Any) -> None:
|
||||
conn_args['autocommit'] = True
|
||||
aconn = await psycopg.AsyncConnection.connect(dsn, **conn_args)
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2024 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Preprocessing of SQL files.
|
||||
"""
|
||||
from typing import Set, Dict, Any, cast
|
||||
import re
|
||||
|
||||
import jinja2
|
||||
|
||||
@@ -34,7 +35,9 @@ def _get_tables(conn: Connection) -> Set[str]:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
|
||||
|
||||
return set((row[0] for row in list(cur)))
|
||||
# paranoia check: make sure we don't get table names that cause
|
||||
# an SQL injection later
|
||||
return {row[0] for row in list(cur) if re.fullmatch(r'\w+', row[0])}
|
||||
|
||||
|
||||
def _get_middle_db_format(conn: Connection, tables: Set[str]) -> str:
|
||||
|
||||
@@ -31,14 +31,19 @@ class Indexer:
|
||||
self.tokenizer = tokenizer
|
||||
self.num_threads = num_threads
|
||||
|
||||
def has_pending(self) -> bool:
|
||||
def has_pending(self, minrank: int = 0, maxrank: int = 30) -> bool:
|
||||
""" Check if any data still needs indexing.
|
||||
This function must only be used after the import has finished.
|
||||
Otherwise it will be very expensive.
|
||||
"""
|
||||
with connect(self.dsn) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 'a' FROM placex WHERE indexed_status > 0 LIMIT 1")
|
||||
cur.execute(""" SELECT 'a'
|
||||
FROM placex
|
||||
WHERE rank_address BETWEEN %s AND %s
|
||||
AND indexed_status > 0
|
||||
LIMIT 1""",
|
||||
(minrank, maxrank))
|
||||
return cur.rowcount > 0
|
||||
|
||||
async def index_full(self, analyse: bool = True) -> None:
|
||||
@@ -56,10 +61,10 @@ class Indexer:
|
||||
cur.execute('ANALYZE')
|
||||
|
||||
while True:
|
||||
if await self.index_by_rank(0, 4) > 0:
|
||||
if await self.index_by_rank(1, 4) > 0:
|
||||
_analyze()
|
||||
|
||||
if await self.index_boundaries(0, 30) > 100:
|
||||
if await self.index_boundaries() > 100:
|
||||
_analyze()
|
||||
|
||||
if await self.index_by_rank(5, 25) > 100:
|
||||
@@ -68,13 +73,16 @@ class Indexer:
|
||||
if await self.index_by_rank(26, 30) > 1000:
|
||||
_analyze()
|
||||
|
||||
# Special case: rank zero depends on the previously-indexed [1..30] ranks
|
||||
await self.index_by_rank(0, 0)
|
||||
|
||||
if await self.index_postcodes() > 100:
|
||||
_analyze()
|
||||
|
||||
if not self.has_pending():
|
||||
break
|
||||
|
||||
async def index_boundaries(self, minrank: int, maxrank: int) -> int:
|
||||
async def index_boundaries(self, minrank: int = 0, maxrank: int = 30) -> int:
|
||||
""" Index only administrative boundaries within the given rank range.
|
||||
"""
|
||||
total = 0
|
||||
@@ -147,8 +155,11 @@ class Indexer:
|
||||
total += await self._index(runners.RankRunner(rank, analyzer),
|
||||
batch=batch, total_tuples=total_tuples.get(rank, 0))
|
||||
|
||||
if maxrank == 30:
|
||||
# Special case: rank zero depends on ranks [1..30]
|
||||
if minrank == 0:
|
||||
total += await self._index(runners.RankRunner(0, analyzer))
|
||||
|
||||
if maxrank == 30:
|
||||
total += await self._index(runners.InterpolationRunner(analyzer), batch=20)
|
||||
|
||||
return total
|
||||
@@ -177,7 +188,7 @@ class Indexer:
|
||||
|
||||
`total_tuples` may contain the total number of rows to process.
|
||||
When not supplied, the value will be computed using the
|
||||
approriate runner function.
|
||||
appropriate runner function.
|
||||
"""
|
||||
LOG.warning("Starting %s (using batch size %s)", runner.name(), batch)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Tokenizer implementing normalisation as used before Nominatim 4 but using
|
||||
@@ -294,13 +294,12 @@ class ICUTokenizer(AbstractTokenizer):
|
||||
with connect(self.dsn) as conn:
|
||||
drop_tables(conn, 'word')
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"ALTER TABLE {old} RENAME TO word")
|
||||
for idx in ('word_token', 'word_id'):
|
||||
cur.execute(f"""ALTER INDEX idx_{old}_{idx}
|
||||
RENAME TO idx_word_{idx}""")
|
||||
for name, _ in WORD_TYPES:
|
||||
cur.execute(f"""ALTER INDEX idx_{old}_{name}
|
||||
RENAME TO idx_word_{name}""")
|
||||
cur.execute(pysql.SQL("ALTER TABLE {} RENAME TO word")
|
||||
.format(pysql.Identifier(old)))
|
||||
for idx in ['word_token', 'word_id'] + [n[0] for n in WORD_TYPES]:
|
||||
cur.execute(pysql.SQL("ALTER INDEX {} RENAME TO {}")
|
||||
.format(pysql.Identifier(f"idx_{old}_{idx}"),
|
||||
pysql.Identifier(f"idx_word_{idx}")))
|
||||
conn.commit()
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2024 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Sanitizer that preprocesses address tags for house numbers. The sanitizer
|
||||
@@ -10,6 +10,7 @@ allows to
|
||||
|
||||
* define which tags are to be considered house numbers (see 'filter-kind')
|
||||
* split house number lists into individual numbers (see 'delimiters')
|
||||
* expand interpolated house numbers
|
||||
|
||||
Arguments:
|
||||
delimiters: Define the set of characters to be used for
|
||||
@@ -23,13 +24,19 @@ Arguments:
|
||||
instead of a house number. Either takes a single string
|
||||
or a list of strings, where each string is a regular
|
||||
expression that must match the full house number value.
|
||||
expand-interpolations: When true, expand house number ranges to separate numbers
|
||||
when an 'interpolation' is present. (default: true)
|
||||
|
||||
"""
|
||||
from typing import Callable, Iterator, List
|
||||
from typing import Callable, Iterator, Iterable, Union
|
||||
import re
|
||||
|
||||
from ...data.place_name import PlaceName
|
||||
from .base import ProcessInfo
|
||||
from .config import SanitizerConfig
|
||||
|
||||
RANGE_REGEX = re.compile(r'\d+-\d+')
|
||||
|
||||
|
||||
class _HousenumberSanitizer:
|
||||
|
||||
@@ -38,21 +45,40 @@ class _HousenumberSanitizer:
|
||||
self.split_regexp = config.get_delimiter()
|
||||
|
||||
self.filter_name = config.get_filter('convert-to-name', 'FAIL_ALL')
|
||||
self.expand_interpolations = config.get_bool('expand-interpolations', True)
|
||||
|
||||
def __call__(self, obj: ProcessInfo) -> None:
|
||||
if not obj.address:
|
||||
return
|
||||
|
||||
new_address: List[PlaceName] = []
|
||||
itype: Union[int, str, None] = None
|
||||
if self.expand_interpolations:
|
||||
itype = next((i.name for i in obj.address if i.kind == 'interpolation'), None)
|
||||
if itype is not None:
|
||||
if itype == 'all':
|
||||
itype = 1
|
||||
elif len(itype) == 1 and itype.isdigit():
|
||||
itype = int(itype)
|
||||
elif itype not in ('odd', 'even'):
|
||||
itype = None
|
||||
|
||||
new_address: list[PlaceName] = []
|
||||
for item in obj.address:
|
||||
if self.filter_kind(item.kind):
|
||||
if itype is not None and RANGE_REGEX.fullmatch(item.name):
|
||||
hnrs = self._expand_range(itype, item.name)
|
||||
if hnrs:
|
||||
new_address.extend(item.clone(kind='housenumber', name=str(hnr))
|
||||
for hnr in hnrs)
|
||||
continue
|
||||
|
||||
if self.filter_name(item.name):
|
||||
obj.names.append(item.clone(kind='housenumber'))
|
||||
else:
|
||||
new_address.extend(item.clone(kind='housenumber', name=n)
|
||||
for n in self.sanitize(item.name))
|
||||
else:
|
||||
# Don't touch other address items.
|
||||
elif item.kind != 'interpolation':
|
||||
# Ignore interpolation, otherwise don't touch other address items.
|
||||
new_address.append(item)
|
||||
|
||||
obj.address = new_address
|
||||
@@ -70,6 +96,22 @@ class _HousenumberSanitizer:
|
||||
def _regularize(self, hnr: str) -> Iterator[str]:
|
||||
yield hnr
|
||||
|
||||
def _expand_range(self, itype: Union[str, int], hnr: str) -> Iterable[int]:
|
||||
first, last = (int(i) for i in hnr.split('-'))
|
||||
|
||||
if isinstance(itype, int):
|
||||
step = itype
|
||||
else:
|
||||
step = 2
|
||||
if (itype == 'even' and first % 2 == 1)\
|
||||
or (itype == 'odd' and first % 2 == 0):
|
||||
first += 1
|
||||
|
||||
if (last + 1 - first) / step < 10:
|
||||
return range(first, last + 1, step)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]:
|
||||
""" Create a housenumber processing function.
|
||||
|
||||
@@ -16,6 +16,7 @@ from psycopg.types.json import Json
|
||||
from ..typing import DictCursorResult
|
||||
from ..config import Configuration
|
||||
from ..db.connection import connect, Cursor, register_hstore
|
||||
from ..db.sql_preprocessor import SQLPreprocessor
|
||||
from ..errors import UsageError
|
||||
from ..tokenizer import factory as tokenizer_factory
|
||||
from ..data.place_info import PlaceInfo
|
||||
@@ -105,3 +106,12 @@ def clean_deleted_relations(config: Configuration, age: str) -> None:
|
||||
except psycopg.DataError as exc:
|
||||
raise UsageError('Invalid PostgreSQL time interval format') from exc
|
||||
conn.commit()
|
||||
|
||||
|
||||
def grant_ro_access(dsn: str, config: Configuration) -> None:
|
||||
""" Grant read-only access to the web user for all Nominatim tables.
|
||||
This can be used to grant access to a different user after import.
|
||||
"""
|
||||
with connect(dsn) as conn:
|
||||
sql = SQLPreprocessor(conn, config)
|
||||
sql.run_sql_file(conn, 'grants.sql')
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Functions for setting up and importing a new Nominatim database.
|
||||
@@ -152,10 +152,11 @@ def create_tables(conn: Connection, config: Configuration, reverse_only: bool =
|
||||
When `reverse_only` is True, then the main table for searching will
|
||||
be skipped and only reverse search is possible.
|
||||
"""
|
||||
sql = SQLPreprocessor(conn, config)
|
||||
sql.env.globals['db']['reverse_only'] = reverse_only
|
||||
SQLPreprocessor(conn, config).run_sql_file(conn, 'tables.sql',
|
||||
create_reverse_only=reverse_only)
|
||||
|
||||
sql.run_sql_file(conn, 'tables.sql')
|
||||
# reinitiate the preprocessor to get all the newly created tables
|
||||
SQLPreprocessor(conn, config).run_sql_file(conn, 'grants.sql')
|
||||
|
||||
|
||||
def create_table_triggers(conn: Connection, config: Configuration) -> None:
|
||||
@@ -193,7 +194,7 @@ def truncate_data_tables(conn: Connection) -> None:
|
||||
WHERE tablename LIKE 'location_road_%'""")
|
||||
|
||||
for table in [r[0] for r in list(cur)]:
|
||||
cur.execute('TRUNCATE ' + table)
|
||||
cur.execute(pysql.SQL('TRUNCATE {}').format(pysql.Identifier(table)))
|
||||
|
||||
conn.commit()
|
||||
|
||||
@@ -218,19 +219,16 @@ async def load_data(dsn: str, threads: int) -> None:
|
||||
pysql.SQL("""INSERT INTO placex ({columns})
|
||||
SELECT {columns} FROM place
|
||||
WHERE osm_id % {total} = {mod}
|
||||
AND NOT (class='place'
|
||||
and (type='houses' or type='postcode'))
|
||||
AND ST_IsValid(geometry)
|
||||
""").format(columns=_COPY_COLUMNS,
|
||||
total=pysql.Literal(placex_threads),
|
||||
mod=pysql.Literal(imod)), None)
|
||||
|
||||
# Interpolations need to be copied seperately
|
||||
# Interpolations need to be copied separately
|
||||
await pool.put_query("""
|
||||
INSERT INTO location_property_osmline (osm_id, address, linegeo)
|
||||
SELECT osm_id, address, geometry FROM place
|
||||
WHERE class='place' and type='houses' and osm_type='W'
|
||||
and ST_GeometryType(geometry) = 'ST_LineString' """, None)
|
||||
INSERT INTO location_property_osmline (osm_id, type, address, linegeo)
|
||||
SELECT osm_id, type, address, geometry
|
||||
FROM place_interpolation
|
||||
""", None)
|
||||
|
||||
progress.cancel()
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ UPDATE_TABLES = [
|
||||
'address_levels',
|
||||
'gb_postcode',
|
||||
'import_osmosis_log',
|
||||
'import_polygon_%',
|
||||
'location_area%',
|
||||
'location_road%',
|
||||
'place',
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Functions for database migration to newer software versions.
|
||||
@@ -29,7 +29,7 @@ _MIGRATION_FUNCTIONS: List[Tuple[NominatimVersion, Callable[..., None]]] = []
|
||||
|
||||
def migrate(config: Configuration, paths: Any) -> int:
|
||||
""" Check for the current database version and execute migrations,
|
||||
if necesssary.
|
||||
if necessary.
|
||||
"""
|
||||
with connect(config.get_libpq_dsn()) as conn:
|
||||
register_hstore(conn)
|
||||
@@ -143,7 +143,7 @@ def create_placex_entrance_table(conn: Connection, config: Configuration, **_: A
|
||||
|
||||
@_migration(5, 1, 99, 1)
|
||||
def create_place_entrance_table(conn: Connection, config: Configuration, **_: Any) -> None:
|
||||
""" Add the place_entrance table to store incomming entrance nodes
|
||||
""" Add the place_entrance table to store incoming entrance nodes
|
||||
"""
|
||||
if not table_exists(conn, 'place_entrance'):
|
||||
with conn.cursor() as cur:
|
||||
@@ -252,7 +252,7 @@ def create_place_postcode_table(conn: Connection, config: Configuration, **_: An
|
||||
""")
|
||||
sqlp.run_string(conn,
|
||||
'GRANT SELECT ON location_postcodes TO "{{config.DATABASE_WEBUSER}}"')
|
||||
# remove postcodes from the various auxillary tables
|
||||
# remove postcodes from the various auxiliary tables
|
||||
cur.execute(
|
||||
"""
|
||||
DELETE FROM place_addressline
|
||||
@@ -350,3 +350,73 @@ def create_place_postcode_table(conn: Connection, config: Configuration, **_: An
|
||||
WHERE osm_type = 'N' and rank_search < 26 and class = 'place';
|
||||
ANALYSE;
|
||||
""")
|
||||
|
||||
|
||||
@_migration(5, 2, 99, 3)
|
||||
def create_place_interpolation_table(conn: Connection, config: Configuration, **_: Any) -> None:
|
||||
""" Create place_interpolation table
|
||||
"""
|
||||
sqlp = SQLPreprocessor(conn, config)
|
||||
mutable = not is_frozen(conn)
|
||||
has_place_table = table_exists(conn, 'place_interpolation')
|
||||
|
||||
if mutable and not has_place_table:
|
||||
# create tables
|
||||
conn.execute("""
|
||||
CREATE TABLE place_interpolation (
|
||||
osm_id BIGINT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
address HSTORE,
|
||||
nodes BIGINT[] NOT NULL,
|
||||
geometry GEOMETRY(LineString, 4326)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS place_interpolation_to_be_deleted (
|
||||
osm_id BIGINT NOT NULL
|
||||
);
|
||||
""")
|
||||
# copy data over
|
||||
conn.execute("""
|
||||
ALTER TABLE place DISABLE TRIGGER ALL;
|
||||
|
||||
WITH deleted AS (
|
||||
DELETE FROM place
|
||||
WHERE class='place' and type = 'houses'
|
||||
RETURNING osm_type, osm_id,
|
||||
address->'interpolation' as itype,
|
||||
address - 'interpolation'::TEXT as address,
|
||||
geometry)
|
||||
INSERT INTO place_interpolation (osm_id, type, address, nodes, geometry)
|
||||
(SELECT d.osm_id, d.itype, d.address, p.nodes, d.geometry
|
||||
FROM deleted d, planet_osm_ways p
|
||||
WHERE osm_type = 'W'
|
||||
AND d.osm_id = p.id
|
||||
AND itype is not null
|
||||
AND ST_GeometryType(geometry) = 'ST_LineString');
|
||||
|
||||
ALTER TABLE place ENABLE TRIGGER ALL;
|
||||
""")
|
||||
|
||||
# create indices
|
||||
conn.execute("""
|
||||
CREATE INDEX place_interpolation_nodes_idx ON place_interpolation
|
||||
USING gin(nodes);
|
||||
CREATE INDEX place_interpolation_osm_id_idx ON place_interpolation
|
||||
USING btree(osm_id);
|
||||
""")
|
||||
# create triggers
|
||||
sqlp.run_sql_file(conn, 'functions/interpolation.sql')
|
||||
conn.execute("""
|
||||
CREATE TRIGGER place_interpolation_before_insert BEFORE INSERT ON place_interpolation
|
||||
FOR EACH ROW EXECUTE PROCEDURE place_interpolation_insert();
|
||||
CREATE TRIGGER place_interpolation_before_delete BEFORE DELETE ON place_interpolation
|
||||
FOR EACH ROW EXECUTE PROCEDURE place_interpolation_delete();
|
||||
""")
|
||||
# mutate location_property_osmline table
|
||||
conn.execute("""
|
||||
ALTER TABLE location_property_osmline ADD COLUMN type TEXT;
|
||||
|
||||
UPDATE location_property_osmline
|
||||
SET type = coalesce(address->'interpolation', 'all'),
|
||||
address = address - 'interpolation'::TEXT;
|
||||
""")
|
||||
|
||||
@@ -78,7 +78,7 @@ class _PostcodeCollector:
|
||||
self.collected[normalized] += (x, y)
|
||||
|
||||
def commit(self, conn: Connection, analyzer: AbstractAnalyzer,
|
||||
project_dir: Optional[Path]) -> None:
|
||||
project_dir: Optional[Path], is_initial: bool) -> None:
|
||||
""" Update postcodes for the country from the postcodes selected so far.
|
||||
|
||||
When 'project_dir' is set, then any postcode files found in this
|
||||
@@ -87,11 +87,14 @@ class _PostcodeCollector:
|
||||
if project_dir is not None:
|
||||
self._update_from_external(analyzer, project_dir)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""SELECT postcode FROM location_postcodes
|
||||
WHERE country_code = %s AND osm_id is null""",
|
||||
(self.country, ))
|
||||
to_delete = [row[0] for row in cur if row[0] not in self.collected]
|
||||
if is_initial:
|
||||
to_delete = []
|
||||
else:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""SELECT postcode FROM location_postcodes
|
||||
WHERE country_code = %s AND osm_id is null""",
|
||||
(self.country, ))
|
||||
to_delete = [row[0] for row in cur if row[0] not in self.collected]
|
||||
|
||||
to_add = [dict(zip(('pc', 'x', 'y'), (k, *v.centroid())))
|
||||
for k, v in self.collected.items()]
|
||||
@@ -102,22 +105,32 @@ class _PostcodeCollector:
|
||||
|
||||
with conn.cursor() as cur:
|
||||
if to_add:
|
||||
cur.executemany(pysql.SQL(
|
||||
"""INSERT INTO location_postcodes
|
||||
(country_code, rank_search, postcode, centroid, geometry)
|
||||
VALUES ({}, {}, %(pc)s,
|
||||
ST_SetSRID(ST_MakePoint(%(x)s, %(y)s), 4326),
|
||||
expand_by_meters(ST_SetSRID(ST_MakePoint(%(x)s, %(y)s), 4326), {}))
|
||||
""").format(pysql.Literal(self.country),
|
||||
pysql.Literal(_extent_to_rank(self.extent)),
|
||||
pysql.Literal(self.extent)),
|
||||
to_add)
|
||||
columns = ['country_code',
|
||||
'rank_search',
|
||||
'postcode',
|
||||
'centroid',
|
||||
'geometry']
|
||||
values = [pysql.Literal(self.country),
|
||||
pysql.Literal(_extent_to_rank(self.extent)),
|
||||
pysql.Placeholder('pc'),
|
||||
pysql.SQL('ST_SetSRID(ST_MakePoint(%(x)s, %(y)s), 4326)'),
|
||||
pysql.SQL("""expand_by_meters(
|
||||
ST_SetSRID(ST_MakePoint(%(x)s, %(y)s), 4326), {})""")
|
||||
.format(pysql.Literal(self.extent))]
|
||||
if is_initial:
|
||||
columns.extend(('place_id', 'indexed_status'))
|
||||
values.extend((pysql.SQL("nextval('seq_place')"), pysql.Literal(1)))
|
||||
|
||||
cur.executemany(pysql.SQL("INSERT INTO location_postcodes ({}) VALUES ({})")
|
||||
.format(pysql.SQL(',')
|
||||
.join(pysql.Identifier(c) for c in columns),
|
||||
pysql.SQL(',').join(values)),
|
||||
to_add)
|
||||
if to_delete:
|
||||
cur.execute("""DELETE FROM location_postcodes
|
||||
WHERE country_code = %s and postcode = any(%s)
|
||||
AND osm_id is null
|
||||
""", (self.country, to_delete))
|
||||
cur.execute("ANALYSE location_postcodes")
|
||||
|
||||
def _update_from_external(self, analyzer: AbstractAnalyzer, project_dir: Path) -> None:
|
||||
""" Look for an external postcode file for the active country in
|
||||
@@ -159,12 +172,13 @@ class _PostcodeCollector:
|
||||
|
||||
if fname.is_file():
|
||||
LOG.info("Using external postcode file '%s'.", fname)
|
||||
return gzip.open(fname, 'rt')
|
||||
return gzip.open(fname, 'rt', encoding='utf-8')
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def update_postcodes(dsn: str, project_dir: Optional[Path], tokenizer: AbstractTokenizer) -> None:
|
||||
def update_postcodes(dsn: str, project_dir: Optional[Path],
|
||||
tokenizer: AbstractTokenizer, force_reimport: bool = False) -> None:
|
||||
""" Update the table of postcodes from the input tables
|
||||
placex and place_postcode.
|
||||
"""
|
||||
@@ -176,45 +190,76 @@ def update_postcodes(dsn: str, project_dir: Optional[Path], tokenizer: AbstractT
|
||||
SET country_code = get_country_code(centroid)
|
||||
WHERE country_code is null
|
||||
""")
|
||||
if force_reimport:
|
||||
conn.execute("TRUNCATE location_postcodes")
|
||||
is_initial = True
|
||||
else:
|
||||
is_initial = _is_postcode_table_empty(conn)
|
||||
if is_initial:
|
||||
conn.execute("""ALTER TABLE location_postcodes
|
||||
DISABLE TRIGGER location_postcodes_before_insert""")
|
||||
# Now update first postcode areas
|
||||
_update_postcode_areas(conn, analyzer, matcher)
|
||||
_update_postcode_areas(conn, analyzer, matcher, is_initial)
|
||||
# Then fill with estimated postcode centroids from other info
|
||||
_update_guessed_postcode(conn, analyzer, matcher, project_dir)
|
||||
_update_guessed_postcode(conn, analyzer, matcher, project_dir, is_initial)
|
||||
if is_initial:
|
||||
conn.execute("""ALTER TABLE location_postcodes
|
||||
ENABLE TRIGGER location_postcodes_before_insert""")
|
||||
conn.commit()
|
||||
|
||||
analyzer.update_postcodes_from_db()
|
||||
|
||||
|
||||
def _is_postcode_table_empty(conn: Connection) -> bool:
|
||||
""" Check if there are any entries in the location_postcodes table yet.
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT place_id FROM location_postcodes LIMIT 1")
|
||||
return cur.fetchone() is None
|
||||
|
||||
|
||||
def _insert_postcode_areas(conn: Connection, country_code: str,
|
||||
extent: int, pcs: list[dict[str, str]]) -> None:
|
||||
extent: int, pcs: list[dict[str, str]],
|
||||
is_initial: bool) -> None:
|
||||
if pcs:
|
||||
with conn.cursor() as cur:
|
||||
columns = ['osm_id', 'country_code',
|
||||
'rank_search', 'postcode',
|
||||
'centroid', 'geometry']
|
||||
values = [pysql.Identifier('osm_id'), pysql.Identifier('country_code'),
|
||||
pysql.Literal(_extent_to_rank(extent)), pysql.Placeholder('out'),
|
||||
pysql.Identifier('centroid'), pysql.Identifier('geometry')]
|
||||
if is_initial:
|
||||
columns.extend(('place_id', 'indexed_status'))
|
||||
values.extend((pysql.SQL("nextval('seq_place')"), pysql.Literal(1)))
|
||||
|
||||
cur.executemany(
|
||||
pysql.SQL(
|
||||
""" INSERT INTO location_postcodes
|
||||
(osm_id, country_code, rank_search, postcode, centroid, geometry)
|
||||
SELECT osm_id, country_code, {}, %(out)s, centroid, geometry
|
||||
FROM place_postcode
|
||||
""" INSERT INTO location_postcodes ({})
|
||||
SELECT {} FROM place_postcode
|
||||
WHERE osm_type = 'R'
|
||||
and country_code = {} and postcode = %(in)s
|
||||
and geometry is not null
|
||||
""").format(pysql.Literal(_extent_to_rank(extent)),
|
||||
""").format(pysql.SQL(',')
|
||||
.join(pysql.Identifier(c) for c in columns),
|
||||
pysql.SQL(',').join(values),
|
||||
pysql.Literal(country_code)),
|
||||
pcs)
|
||||
|
||||
|
||||
def _update_postcode_areas(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
matcher: PostcodeFormatter) -> None:
|
||||
matcher: PostcodeFormatter, is_initial: bool) -> None:
|
||||
""" Update the postcode areas made from postcode boundaries.
|
||||
"""
|
||||
# first delete all areas that have gone
|
||||
conn.execute(""" DELETE FROM location_postcodes pc
|
||||
WHERE pc.osm_id is not null
|
||||
AND NOT EXISTS(
|
||||
SELECT * FROM place_postcode pp
|
||||
WHERE pp.osm_type = 'R' and pp.osm_id = pc.osm_id
|
||||
and geometry is not null)
|
||||
""")
|
||||
if not is_initial:
|
||||
conn.execute(""" DELETE FROM location_postcodes pc
|
||||
WHERE pc.osm_id is not null
|
||||
AND NOT EXISTS(
|
||||
SELECT * FROM place_postcode pp
|
||||
WHERE pp.osm_type = 'R' and pp.osm_id = pc.osm_id
|
||||
and geometry is not null)
|
||||
""")
|
||||
# now insert all in country batches, triggers will ensure proper updates
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(""" SELECT country_code, postcode FROM place_postcode
|
||||
@@ -230,7 +275,8 @@ def _update_postcode_areas(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
fmt = matcher.get_matcher(country_code)
|
||||
elif country_code != cc:
|
||||
_insert_postcode_areas(conn, country_code,
|
||||
matcher.get_postcode_extent(country_code), pcs)
|
||||
matcher.get_postcode_extent(country_code), pcs,
|
||||
is_initial)
|
||||
country_code = cc
|
||||
fmt = matcher.get_matcher(country_code)
|
||||
pcs = []
|
||||
@@ -241,21 +287,26 @@ def _update_postcode_areas(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
|
||||
if country_code is not None and pcs:
|
||||
_insert_postcode_areas(conn, country_code,
|
||||
matcher.get_postcode_extent(country_code), pcs)
|
||||
matcher.get_postcode_extent(country_code), pcs,
|
||||
is_initial)
|
||||
|
||||
|
||||
def _update_guessed_postcode(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
matcher: PostcodeFormatter, project_dir: Optional[Path]) -> None:
|
||||
matcher: PostcodeFormatter, project_dir: Optional[Path],
|
||||
is_initial: bool) -> None:
|
||||
""" Computes artificial postcode centroids from the placex table,
|
||||
potentially enhances it with external data and then updates the
|
||||
postcodes in the table 'location_postcodes'.
|
||||
"""
|
||||
# First get the list of countries that currently have postcodes.
|
||||
# (Doing this before starting to insert, so it is fast on import.)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""SELECT DISTINCT country_code FROM location_postcodes
|
||||
WHERE osm_id is null""")
|
||||
todo_countries = {row[0] for row in cur}
|
||||
if is_initial:
|
||||
todo_countries: set[str] = set()
|
||||
else:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""SELECT DISTINCT country_code FROM location_postcodes
|
||||
WHERE osm_id is null""")
|
||||
todo_countries = {row[0] for row in cur}
|
||||
|
||||
# Next, get the list of postcodes that are already covered by areas.
|
||||
area_pcs = defaultdict(set)
|
||||
@@ -275,6 +326,7 @@ def _update_guessed_postcode(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
FROM place_postcode WHERE geometry is not null)
|
||||
""")
|
||||
cur.execute("CREATE INDEX ON _global_postcode_area USING gist(geometry)")
|
||||
|
||||
# Recompute the list of valid postcodes from placex.
|
||||
with conn.cursor(name="placex_postcodes") as cur:
|
||||
cur.execute("""
|
||||
@@ -296,7 +348,7 @@ def _update_guessed_postcode(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
for country, postcode, x, y in cur:
|
||||
if collector is None or country != collector.country:
|
||||
if collector is not None:
|
||||
collector.commit(conn, analyzer, project_dir)
|
||||
collector.commit(conn, analyzer, project_dir, is_initial)
|
||||
collector = _PostcodeCollector(country, matcher.get_matcher(country),
|
||||
matcher.get_postcode_extent(country),
|
||||
exclude=area_pcs[country])
|
||||
@@ -304,14 +356,14 @@ def _update_guessed_postcode(conn: Connection, analyzer: AbstractAnalyzer,
|
||||
collector.add(postcode, x, y)
|
||||
|
||||
if collector is not None:
|
||||
collector.commit(conn, analyzer, project_dir)
|
||||
collector.commit(conn, analyzer, project_dir, is_initial)
|
||||
|
||||
# Now handle any countries that are only in the postcode table.
|
||||
for country in todo_countries:
|
||||
fmt = matcher.get_matcher(country)
|
||||
ext = matcher.get_postcode_extent(country)
|
||||
_PostcodeCollector(country, fmt, ext,
|
||||
exclude=area_pcs[country]).commit(conn, analyzer, project_dir)
|
||||
exclude=area_pcs[country]).commit(conn, analyzer, project_dir, False)
|
||||
|
||||
conn.execute("DROP TABLE IF EXISTS _global_postcode_area")
|
||||
|
||||
|
||||
@@ -141,7 +141,9 @@ def import_importance_csv(dsn: str, data_file: Path) -> int:
|
||||
|
||||
copy_cmd = """COPY wikimedia_importance(language, title, importance, wikidata)
|
||||
FROM STDIN"""
|
||||
with gzip.open(str(data_file), 'rt') as fd, cur.copy(copy_cmd) as copy:
|
||||
with gzip.open(
|
||||
str(data_file), 'rt', encoding='utf-8') as fd, \
|
||||
cur.copy(copy_cmd) as copy:
|
||||
for row in csv.DictReader(fd, delimiter='\t', quotechar='|'):
|
||||
wd_id = int(row['wikidata_id'][1:])
|
||||
copy.write_row((row['language'],
|
||||
|
||||
@@ -17,13 +17,12 @@ import tarfile
|
||||
from psycopg.types.json import Json
|
||||
|
||||
from ..config import Configuration
|
||||
from ..db.connection import connect
|
||||
from ..db.connection import connect, table_exists
|
||||
from ..db.sql_preprocessor import SQLPreprocessor
|
||||
from ..errors import UsageError
|
||||
from ..db.query_pool import QueryPool
|
||||
from ..data.place_info import PlaceInfo
|
||||
from ..tokenizer.base import AbstractTokenizer
|
||||
from . import freeze
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
@@ -90,16 +89,19 @@ async def add_tiger_data(data_dir: str, config: Configuration, threads: int,
|
||||
"""
|
||||
dsn = config.get_libpq_dsn()
|
||||
|
||||
with connect(dsn) as conn:
|
||||
if freeze.is_frozen(conn):
|
||||
raise UsageError("Tiger cannot be imported when database frozen (Github issue #3048)")
|
||||
|
||||
with TigerInput(data_dir) as tar:
|
||||
if not tar:
|
||||
return 1
|
||||
|
||||
with connect(dsn) as conn:
|
||||
sql = SQLPreprocessor(conn, config)
|
||||
|
||||
if not table_exists(conn, 'search_name'):
|
||||
raise UsageError(
|
||||
"Cannot perform tiger import: required tables are missing. "
|
||||
"See https://github.com/osm-search/Nominatim/issues/2463 for details."
|
||||
)
|
||||
|
||||
sql.run_sql_file(conn, 'tiger_import_start.sql')
|
||||
|
||||
# Reading files and then for each file line handling
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Version information for Nominatim.
|
||||
@@ -55,7 +55,7 @@ def parse_version(version: str) -> NominatimVersion:
|
||||
return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')])
|
||||
|
||||
|
||||
NOMINATIM_VERSION = parse_version('5.2.99-2')
|
||||
NOMINATIM_VERSION = parse_version('5.2.99-3')
|
||||
|
||||
POSTGRESQL_REQUIRED_VERSION = (12, 0)
|
||||
POSTGIS_REQUIRED_VERSION = (3, 0)
|
||||
|
||||
@@ -42,6 +42,22 @@ Feature: Tests for finding places by osm_type and osm_id
|
||||
| jsonv2 | json |
|
||||
| geojson | geojson |
|
||||
|
||||
Scenario Outline: Lookup with entrances
|
||||
When sending v1/lookup with format <format>
|
||||
| osm_ids | entrances |
|
||||
| W429210603 | 1 |
|
||||
Then a HTTP 200 is returned
|
||||
And the result is valid <outformat>
|
||||
And result 0 contains in field entrances+0
|
||||
| osm_id | type | lat | lon |
|
||||
| 6580031131 | yes | 47.2489382 | 9.5284033 |
|
||||
|
||||
Examples:
|
||||
| format | outformat |
|
||||
| json | json |
|
||||
| jsonv2 | json |
|
||||
| geojson | geojson |
|
||||
|
||||
Scenario: Linked places return information from the linkee
|
||||
When sending v1/lookup with format geocodejson
|
||||
| osm_ids |
|
||||
|
||||
@@ -167,3 +167,18 @@ Feature: v1/reverse Parameter Tests
|
||||
| json | json |
|
||||
| jsonv2 | json |
|
||||
| xml | xml |
|
||||
|
||||
Scenario Outline: Reverse with entrances
|
||||
When sending v1/reverse with format <format>
|
||||
| lat | lon | entrances | zoom |
|
||||
| 47.24942041089678 | 9.52854573737568 | 1 | 18 |
|
||||
Then a HTTP 200 is returned
|
||||
And the result is valid <outformat>
|
||||
And the result contains array field entrances where element 0 contains
|
||||
| osm_id | type | lat | lon |
|
||||
| 6580031131 | yes | 47.2489382 | 9.5284033 |
|
||||
|
||||
Examples:
|
||||
| format | outformat |
|
||||
| json | json |
|
||||
| jsonv2 | json |
|
||||
|
||||
@@ -241,8 +241,8 @@ Feature: Address computation
|
||||
Scenario: buildings with only addr:postcodes do not appear in the address of a way
|
||||
Given the grid with origin DE
|
||||
| 1 | | | | | 8 | | 6 | | 2 |
|
||||
| |10 |11 | | | | | | | |
|
||||
| |13 |12 | | | | | | | |
|
||||
| | | | | | | | | | |
|
||||
| |13 | | | | | | | | |
|
||||
| 20| | | 21| | | | | | |
|
||||
| | | | | | | | | | |
|
||||
| | | | | | 9 | | | | |
|
||||
@@ -255,9 +255,9 @@ Feature: Address computation
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W93 | highway | residential | 20,21 |
|
||||
And the places
|
||||
| osm | class | type | addr+postcode | geometry |
|
||||
| W22 | place | postcode | 11234 | (10,11,12,13,10) |
|
||||
And the postcodes
|
||||
| osm | postcode | centroid |
|
||||
| W22 | 11234 | 13 |
|
||||
When importing
|
||||
Then place_addressline contains exactly
|
||||
| object | address |
|
||||
|
||||
@@ -8,12 +8,9 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to no interpolation
|
||||
|
||||
@@ -25,15 +22,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2 | 1,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -47,15 +41,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 8 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 2,1 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 2,1 | 2,1 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 2,1 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -69,15 +60,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 1 |
|
||||
| N2 | place | house | 11 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | odd | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | odd | 1,2 | 1,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -91,15 +79,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 1 |
|
||||
| N2 | place | house | 4 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | all | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | all | 1,2 | 1,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -113,15 +98,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 12 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,3,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,3,2 | 1,3,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -135,15 +117,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 10 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,3,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,3,2 | 1,3,3,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3,3,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -158,15 +137,12 @@ Feature: Import of address interpolations
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 14 |
|
||||
| N3 | place | house | 10 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,3,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,3,2 | 1,3,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -184,15 +160,12 @@ Feature: Import of address interpolations
|
||||
| N2 | place | house | 14 |
|
||||
| N3 | place | house | 10 |
|
||||
| N4 | place | house | 18 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,3,2,4 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,3,2,4 | 1,3,2,4 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 1,3,2,4 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3,2,4 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -209,15 +182,12 @@ Feature: Import of address interpolations
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 14 |
|
||||
| N3 | place | house | 10 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 2,3,1 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 2,3,1 | 2,3,1 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 2,3,1 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -233,15 +203,12 @@ Feature: Import of address interpolations
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 8 |
|
||||
| N3 | place | house | 7 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,3,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,3,2 | 1,3,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -257,15 +224,12 @@ Feature: Import of address interpolations
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
| N3 | place | house | 10 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2,3,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2,3,2 | 1,2,3,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 1,2,3 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2,3,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -281,15 +245,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2,3,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2,3,2 | 1,2,3,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 1,2,3 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2,3,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -306,18 +267,14 @@ Feature: Import of address interpolations
|
||||
| N2 | place | house | 6 | 2 |
|
||||
| N3 | place | house | 12 | 1 |
|
||||
| N4 | place | house | 16 | 2 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | street | geometry |
|
||||
| W10 | place | houses | even | | 1,2 |
|
||||
| W11 | place | houses | even | Cloud Street | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | street | nodes | geometry | nodes |
|
||||
| W10 | even | | 1,2 | 1,2 | 1,2 |
|
||||
| W11 | even | Cloud Street | 3,4 | 1,2 | 3,4 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | tertiary | Sun Way | 10,11 |
|
||||
| W3 | highway | tertiary | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
| 11 | 3,4 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -351,18 +308,14 @@ Feature: Import of address interpolations
|
||||
| N2 | place | house | 6 | | 2 |
|
||||
| N3 | place | house | 12 | Cloud Street | 1 |
|
||||
| N4 | place | house | 16 | Cloud Street | 2 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W10 | place | houses | even | 1,2 |
|
||||
| W11 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W10 | even | 1,2 | 1,2 |
|
||||
| W11 | even | 1,2 | 3,4 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | tertiary | Sun Way | 10,11 |
|
||||
| W3 | highway | tertiary | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
| 11 | 3,4 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -391,15 +344,12 @@ Feature: Import of address interpolations
|
||||
| N1 | place | house | 10 | 144.9632341 -37.76163 |
|
||||
| N2 | place | house | 6 | 144.9630541 -37.7628174 |
|
||||
| N3 | shop | supermarket | 2 | 144.9629794 -37.7630755 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 144.9632341 -37.76163,144.9630541 -37.7628172,144.9629794 -37.7630755 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 144.9632341 -37.76163,144.9630541 -37.7628172,144.9629794 -37.7630755 | 1,2,3 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 144.9632341 -37.76163,144.9629794 -37.7630755 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2,3 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -415,24 +365,21 @@ Feature: Import of address interpolations
|
||||
| N1 | place | house | 23 |
|
||||
| N2 | amenity | school | |
|
||||
| N3 | place | house | 29 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | odd | 1,2,3 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | odd | 1,2,3 | 1,2,3 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2,3 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
| 25 | 27 | 0.0000166 0,0.00002 0,0.0000333 0 |
|
||||
|
||||
Scenario: Ways without node entries are ignored
|
||||
Given the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| W1 | place | houses | even | 1 1, 1 1.001 |
|
||||
Given the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1 1, 1 1.001 | 34,45 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 1 1, 1 1.001 |
|
||||
@@ -447,9 +394,9 @@ Feature: Import of address interpolations
|
||||
| osm | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
Given the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| W1 | place | houses | even | 1,2 |
|
||||
Given the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2 | 1,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
@@ -464,15 +411,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 0 |
|
||||
| N2 | place | house | 10 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W10 | highway | residential | London Road |4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -497,12 +441,9 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | name | geometry |
|
||||
| W1 | highway | residential | Vert St | 1,2 |
|
||||
| W2 | highway | residential | Horiz St | 2,3 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | addr+inclusion | geometry |
|
||||
| W10 | place | houses | even | actual | 8,9 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 8,9 |
|
||||
And the interpolations
|
||||
| osm | type | addr+inclusion | geometry | nodes |
|
||||
| W10 | even | actual | 8,9 | 8,9 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -521,15 +462,12 @@ Feature: Import of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | <value> | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | <value> | 1,2 | 1,2 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to no interpolation
|
||||
|
||||
@@ -549,15 +487,12 @@ Feature: Import of address interpolations
|
||||
| N2 | place | house | 18 | 3 |
|
||||
| N3 | place | house | 24 | 9 |
|
||||
| N4 | place | house | 42 | 4 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2,3,4 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2,3,4 | 1,2,3,4 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 1,4 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2,3,4 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end |
|
||||
@@ -576,15 +511,12 @@ Feature: Import of address interpolations
|
||||
| N2 | place | house | 6 | 8 |
|
||||
| N3 | place | house | 10 | 8 |
|
||||
| N4 | place | house | 14 | 9 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 7,8,8,9 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 7,8,8,9 | 1,2,3,4 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 4,5 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2,3,4 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
@@ -601,15 +533,12 @@ Feature: Import of address interpolations
|
||||
| N2 | place | house | 8 |
|
||||
| N3 | place | house | 12 |
|
||||
| N4 | place | house | 14 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 8,9 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 8,9 | 1,8,9,2,3,4 |
|
||||
And the named places
|
||||
| osm | class | type | geometry |
|
||||
| W10 | highway | residential | 1,4 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,8,9,2,3,4 |
|
||||
When importing
|
||||
Then W1 expands to interpolation
|
||||
| start | end | geometry |
|
||||
|
||||
@@ -306,11 +306,17 @@ Feature: Linking of places
|
||||
Given the places
|
||||
| osm | class | type | name+name | geometry |
|
||||
| N9 | place | city | Popayán | 9 |
|
||||
| R1 | boundary | administrative | Perímetro Urbano Popayán | (1,2,3,4,1) |
|
||||
Given the places
|
||||
| osm | class | type | name+name | geometry | admin |
|
||||
| R1 | boundary | administrative | Perímetro Urbano Popayán | (1,2,3,4,1) | 8 |
|
||||
And the relations
|
||||
| id | members |
|
||||
| 1 | N9:label |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | linked_place_id |
|
||||
| N9:place | R1 |
|
||||
| R1:boundary | - |
|
||||
Then placex contains
|
||||
| object | name+_place_name | name+_place_name:es |
|
||||
| R1 | Popayán | Popayán |
|
||||
|
||||
@@ -287,34 +287,62 @@ Feature: Searching of house numbers
|
||||
| N1 |
|
||||
|
||||
|
||||
Scenario: Interpolations are found according to their type
|
||||
Given the grid
|
||||
| 10 | | 11 |
|
||||
| 100 | | 101 |
|
||||
| 20 | | 21 |
|
||||
Scenario: A housenumber with interpolation is found
|
||||
Given the places
|
||||
| osm | class | type | housenr | addr+interpolation | geometry |
|
||||
| N1 | building | yes | 1-5 | odd | 9 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W100 | highway | residential | Ringstr | 100, 101 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W10 | place | houses | even | 10, 11 |
|
||||
| W20 | place | houses | odd | 20, 21 |
|
||||
And the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| N10 | place | house | 10 | 10 |
|
||||
| N11 | place | house | 20 | 11 |
|
||||
| N20 | place | house | 11 | 20 |
|
||||
| N21 | place | house | 21 | 21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 10, 11 |
|
||||
| 20 | 20, 21 |
|
||||
| osm | class | type | name | geometry |
|
||||
| W10 | highway | path | Rue Paris | 1,2,3 |
|
||||
When importing
|
||||
When geocoding "Ringstr 12"
|
||||
When geocoding "Rue Paris 1"
|
||||
Then the result set contains
|
||||
| object | address+house_number |
|
||||
| N1 | 1-5 |
|
||||
When geocoding "Rue Paris 3"
|
||||
Then the result set contains
|
||||
| object | address+house_number |
|
||||
| N1 | 1-5 |
|
||||
When geocoding "Rue Paris 5"
|
||||
Then the result set contains
|
||||
| object | address+house_number |
|
||||
| N1 | 1-5 |
|
||||
When geocoding "Rue Paris 2"
|
||||
Then the result set contains
|
||||
| object |
|
||||
| W10 |
|
||||
When geocoding "Ringstr 13"
|
||||
| W10 |
|
||||
|
||||
Scenario: A housenumber with bad interpolation is ignored
|
||||
Given the places
|
||||
| osm | class | type | housenr | addr+interpolation | geometry |
|
||||
| N1 | building | yes | 1-5 | bad | 9 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W10 | highway | path | Rue Paris | 1,2,3 |
|
||||
When importing
|
||||
When geocoding "Rue Paris 1-5"
|
||||
Then the result set contains
|
||||
| object | address+house_number |
|
||||
| N1 | 1-5 |
|
||||
When geocoding "Rue Paris 3"
|
||||
Then the result set contains
|
||||
| object |
|
||||
| W20 |
|
||||
| W10 |
|
||||
|
||||
|
||||
Scenario: A bad housenumber with a good interpolation is just a housenumber
|
||||
Given the places
|
||||
| osm | class | type | housenr | addr+interpolation | geometry |
|
||||
| N1 | building | yes | 1-100 | all | 9 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W10 | highway | path | Rue Paris | 1,2,3 |
|
||||
When importing
|
||||
When geocoding "Rue Paris 1-100"
|
||||
Then the result set contains
|
||||
| object | address+house_number |
|
||||
| N1 | 1-100 |
|
||||
When geocoding "Rue Paris 3"
|
||||
Then the result set contains
|
||||
| object |
|
||||
| W10 |
|
||||
|
||||
@@ -11,16 +11,13 @@ Feature: Query of address interpolations
|
||||
Given the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W10 | highway | primary | Nickway | 10,12,13 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | odd | 1,3 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | odd | 1,3 | 1,3 |
|
||||
And the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| N1 | place | house | 1 | 1 |
|
||||
| N3 | place | house | 5 | 3 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3 |
|
||||
When importing
|
||||
When reverse geocoding at node 2
|
||||
Then the result contains
|
||||
@@ -36,16 +33,13 @@ Feature: Query of address interpolations
|
||||
Given the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W10 | highway | primary | Nickway | 10,12,13 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,3 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,3 | 1,3 |
|
||||
And the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| N1 | place | house | 2 | 1 |
|
||||
| N3 | place | house | 18 | 3 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,3 |
|
||||
When importing
|
||||
When reverse geocoding at node 2
|
||||
Then the result contains
|
||||
@@ -55,3 +49,32 @@ Feature: Query of address interpolations
|
||||
Then all results contain
|
||||
| object | display_name | centroid!wkt |
|
||||
| W1 | 10, Nickway | 2 |
|
||||
|
||||
|
||||
Scenario: Interpolations are found according to their type
|
||||
Given the grid
|
||||
| 10 | | 11 |
|
||||
| 100 | | 101 |
|
||||
| 20 | | 21 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W100 | highway | residential | Ringstr | 100, 101 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W10 | even | 10, 11 | 10, 11 |
|
||||
| W20 | odd | 20, 21 | 20, 21 |
|
||||
And the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| N10 | place | house | 10 | 10 |
|
||||
| N11 | place | house | 20 | 11 |
|
||||
| N20 | place | house | 11 | 20 |
|
||||
| N21 | place | house | 21 | 21 |
|
||||
When importing
|
||||
When geocoding "Ringstr 12"
|
||||
Then the result set contains
|
||||
| object |
|
||||
| W10 |
|
||||
When geocoding "Ringstr 13"
|
||||
Then the result set contains
|
||||
| object |
|
||||
| W20 |
|
||||
|
||||
@@ -80,3 +80,23 @@ Feature: Searching of simple objects
|
||||
| Chicago | Illinois | IL |
|
||||
| Auburn | Alabama | AL |
|
||||
| New Orleans | Louisiana | LA |
|
||||
|
||||
# github #3210
|
||||
Scenario: Country with alternate-language name does not dominate when locale differs
|
||||
Given the 1.0 grid with origin DE
|
||||
| 1 | | 2 |
|
||||
| | 10 | |
|
||||
| 4 | | 3 |
|
||||
Given the places
|
||||
| osm | class | type | admin | name+name | name+name:fi | name+name:de | country | geometry |
|
||||
| R1 | boundary | administrative | 2 | Turgei | Turgi | Testland | de | (1,2,3,4,1) |
|
||||
Given the places
|
||||
| osm | class | type | name+name | geometry |
|
||||
| N10 | place | village | Turgi | 10 |
|
||||
When importing
|
||||
And geocoding "Turgi"
|
||||
| accept-language |
|
||||
| de |
|
||||
Then result 0 contains
|
||||
| object |
|
||||
| N10 |
|
||||
|
||||
@@ -11,18 +11,15 @@ Feature: Update of address interpolations
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Sun Way | 10,11 |
|
||||
| W3 | highway | unclassified | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
When importing
|
||||
Then W10 expands to no interpolation
|
||||
When updating places
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And updating places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W10 | place | houses | even | 1,2 |
|
||||
And updating interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W10 | even | 1,2 | 1,2 |
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
| N1 | W2 |
|
||||
@@ -41,16 +38,13 @@ Feature: Update of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W10 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W10 | even | 1,2 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Sun Way | 10,11 |
|
||||
| W3 | highway | unclassified | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -59,9 +53,9 @@ Feature: Update of address interpolations
|
||||
And W10 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W2 | 4 | 4 |
|
||||
When updating places
|
||||
| osm | class | type | addr+interpolation | street | geometry |
|
||||
| W10 | place | houses | even | Cloud Street | 1,2 |
|
||||
When updating interpolations
|
||||
| osm | type | street | nodes | geometry |
|
||||
| W10 | even | Cloud Street | 1,2 | 1,2 |
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
| N1 | W3 |
|
||||
@@ -80,16 +74,13 @@ Feature: Update of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W10 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W10 | even | 1,2 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Sun Way | 10,11 |
|
||||
| W3 | highway | unclassified | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -120,16 +111,13 @@ Feature: Update of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W10 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W10 | even | 1,2 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Sun Way | 10,11 |
|
||||
| W3 | highway | unclassified | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -155,15 +143,12 @@ Feature: Update of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | street | geometry |
|
||||
| W10 | place | houses | even | Cloud Street| 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | street | geometry | nodes |
|
||||
| W10 | even | Cloud Street| 1,2 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Sun Way | 10,11 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -193,16 +178,13 @@ Feature: Update of address interpolations
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | street | geometry |
|
||||
| W10 | place | houses | even | Cloud Street| 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | street | geometry | nodes |
|
||||
| W10 | even | Cloud Street| 1,2 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Sun Way | 10,11 |
|
||||
| W3 | highway | unclassified | Cloud Street | 20,21 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 10 | 1,2 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
@@ -220,67 +202,6 @@ Feature: Update of address interpolations
|
||||
| parent_place_id | start | end |
|
||||
| W2 | 4 | 4 |
|
||||
|
||||
Scenario: building becomes interpolation
|
||||
Given the grid
|
||||
| 10 | | | | 11 |
|
||||
| | 1 | | 2 | |
|
||||
| | 4 | | 3 | |
|
||||
And the places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| W1 | place | house | 3 | (1,2,3,4,1) |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Cloud Street | 10,11 |
|
||||
When importing
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
| W1 | W2 |
|
||||
Given the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
When updating places
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And updating places
|
||||
| osm | class | type | addr+interpolation | street | geometry |
|
||||
| W1 | place | houses | even | Cloud Street| 1,2 |
|
||||
Then placex has no entry for W1
|
||||
And W1 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W2 | 4 | 4 |
|
||||
|
||||
Scenario: interpolation becomes building
|
||||
Given the grid
|
||||
| 10 | | | | 11 |
|
||||
| | 1 | | 2 | |
|
||||
| | 4 | | 3 | |
|
||||
And the places
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Cloud Street | 10,11 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | street | geometry |
|
||||
| W1 | place | houses | even | Cloud Street| 1,2 |
|
||||
When importing
|
||||
Then placex has no entry for W1
|
||||
And W1 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W2 | 4 | 4 |
|
||||
When updating places
|
||||
| osm | class | type | housenr | geometry |
|
||||
| W1 | place | house | 3 | (1,2,3,4,1) |
|
||||
Then placex contains
|
||||
| object | parent_place_id |
|
||||
| W1 | W2 |
|
||||
And W1 expands to no interpolation
|
||||
|
||||
Scenario: housenumbers added to interpolation
|
||||
Given the grid
|
||||
| 10 | | | | 11 |
|
||||
@@ -288,18 +209,18 @@ Feature: Update of address interpolations
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W2 | highway | unclassified | Cloud Street | 10,11 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 1 | 1,2 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W1 | place | houses | even | 1,2 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2 | 1,2 |
|
||||
When importing
|
||||
Then W1 expands to no interpolation
|
||||
When updating places
|
||||
| osm | class | type | housenr |
|
||||
| N1 | place | house | 2 |
|
||||
| N2 | place | house | 6 |
|
||||
And updating interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W1 | even | 1,2 | 1,2 |
|
||||
Then W1 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W2 | 4 | 4 |
|
||||
@@ -311,12 +232,9 @@ Feature: Update of address interpolations
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W1 | highway | unclassified | Cloud Street | 1, 2 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 2 | 3,4,5 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W2 | place | houses | even | 3,4,5 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W2 | even | 3,4,5 | 3,4,5 |
|
||||
And the places
|
||||
| osm | class | type | housenr |
|
||||
| N3 | place | house | 2 |
|
||||
@@ -328,12 +246,14 @@ Feature: Update of address interpolations
|
||||
When updating places
|
||||
| osm | class | type | housenr |
|
||||
| N4 | place | house | 6 |
|
||||
And updating interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W2 | even | 3,4,5 | 3,4,5 |
|
||||
Then W2 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W1 | 4 | 4 |
|
||||
| W1 | 8 | 8 |
|
||||
|
||||
@skip
|
||||
Scenario: housenumber removed in middle of interpolation
|
||||
Given the grid
|
||||
| 1 | | | | | 2 |
|
||||
@@ -341,12 +261,9 @@ Feature: Update of address interpolations
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W1 | highway | unclassified | Cloud Street | 1, 2 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 2 | 3,4,5 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W2 | place | houses | even | 3,4,5 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W2 | even | 3,4,5 | 3,4,5 |
|
||||
And the places
|
||||
| osm | class | type | housenr |
|
||||
| N3 | place | house | 2 |
|
||||
@@ -358,6 +275,9 @@ Feature: Update of address interpolations
|
||||
| W1 | 4 | 4 |
|
||||
| W1 | 8 | 8 |
|
||||
When marking for delete N4
|
||||
And updating interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W2 | even | 3,4,5 | 3,4,5 |
|
||||
Then W2 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W1 | 4 | 8 |
|
||||
@@ -369,12 +289,9 @@ Feature: Update of address interpolations
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W1 | highway | unclassified | Cloud Street | 1, 2 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 2 | 3,4 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W2 | place | houses | even | 3,4 |
|
||||
And the interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W2 | even | 3,4 | 3,4 |
|
||||
And the places
|
||||
| osm | class | type | housenr |
|
||||
| N3 | place | house | 2 |
|
||||
@@ -386,33 +303,9 @@ Feature: Update of address interpolations
|
||||
When updating places
|
||||
| osm | class | type | housenr |
|
||||
| N4 | place | house | 8 |
|
||||
And updating interpolations
|
||||
| osm | type | geometry | nodes |
|
||||
| W2 | even | 3,4 | 3,4 |
|
||||
Then W2 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W1 | 4 | 6 |
|
||||
|
||||
Scenario: Legal interpolation type changed to illegal one
|
||||
Given the grid
|
||||
| 1 | | 2 |
|
||||
| 3 | | 4 |
|
||||
And the places
|
||||
| osm | class | type | name | geometry |
|
||||
| W1 | highway | unclassified | Cloud Street | 1, 2 |
|
||||
And the ways
|
||||
| id | nodes |
|
||||
| 2 | 3,4 |
|
||||
And the places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W2 | place | houses | even | 3,4 |
|
||||
And the places
|
||||
| osm | class | type | housenr |
|
||||
| N3 | place | house | 2 |
|
||||
| N4 | place | house | 6 |
|
||||
When importing
|
||||
Then W2 expands to interpolation
|
||||
| parent_place_id | start | end |
|
||||
| W1 | 4 | 4 |
|
||||
When updating places
|
||||
| osm | class | type | addr+interpolation | geometry |
|
||||
| W2 | place | houses | 12-2 | 3,4 |
|
||||
Then W2 expands to no interpolation
|
||||
|
||||
|
||||
@@ -68,19 +68,6 @@ Feature: Update of simple objects
|
||||
| object | class | type | centroid!wkt |
|
||||
| N3 | shop | grocery | 1 -1 |
|
||||
|
||||
Scenario: remove postcode place when house number is added
|
||||
Given the places
|
||||
| osm | class | type | postcode | geometry |
|
||||
| N3 | place | postcode | 12345 | country:de |
|
||||
When importing
|
||||
Then placex has no entry for N3
|
||||
When updating places
|
||||
| osm | class | type | postcode | housenr | geometry |
|
||||
| N3 | place | house | 12345 | 13 | country:de |
|
||||
Then placex contains
|
||||
| object | class | type |
|
||||
| N3 | place | house |
|
||||
|
||||
Scenario: remove boundary when changing from polygon to way
|
||||
Given the grid
|
||||
| 1 | 2 |
|
||||
|
||||
42
test/bdd/features/osm2pgsql/import/interpolation.feature
Normal file
42
test/bdd/features/osm2pgsql/import/interpolation.feature
Normal file
@@ -0,0 +1,42 @@
|
||||
Feature: Import of interpolations
|
||||
Test if interpolation objects are correctly imported into the
|
||||
place_interpolation table
|
||||
|
||||
Background:
|
||||
Given the grid
|
||||
| 1 | 2 |
|
||||
| 4 | 3 |
|
||||
|
||||
Scenario: Simple address interpolations
|
||||
When loading osm data
|
||||
"""
|
||||
n1
|
||||
n2
|
||||
w13001 Taddr:interpolation=odd,addr:street=Blumenstrasse Nn1,n2
|
||||
w13002 Taddr:interpolation=even,place=city Nn1,n2
|
||||
w13003 Taddr:interpolation=odd Nn1,n1
|
||||
"""
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| W13002 | place | city |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type | address!dict | nodes!ints | geometry!wkt |
|
||||
| 13001 | odd | "street": "Blumenstrasse" | 1,2 | 1,2 |
|
||||
| 13002 | even | - | 1,2 | 1,2 |
|
||||
|
||||
Scenario: Address interpolation with housenumber
|
||||
When loading osm data
|
||||
"""
|
||||
n1
|
||||
n2
|
||||
n3
|
||||
n4
|
||||
w34 Taddr:interpolation=all,addr:housenumber=2-4,building=yes Nn1,n2,n3,n4,n1
|
||||
w35 Taddr:interpolation=all,addr:housenumber=5,building=yes Nn1,n2,n3,n4,n1
|
||||
w36 Taddr:interpolation=all,addr:housenumber=2a-c Nn1,n2,n3,n4,n1
|
||||
"""
|
||||
Then place contains exactly
|
||||
| object | class | type | address!dict |
|
||||
| W35 | building | yes | "housenumber": "5", "interpolation": "all" |
|
||||
| W34 | building | yes | "housenumber": "2-4", "interpolation": "all" |
|
||||
| W36 | place | house | "housenumber": "2a-c", "interpolation": "all" |
|
||||
@@ -205,18 +205,6 @@ Feature: Tag evaluation
|
||||
| N12005 | 12345 | - |
|
||||
|
||||
|
||||
Scenario: Address interpolations
|
||||
When loading osm data
|
||||
"""
|
||||
n13001 Taddr:interpolation=odd
|
||||
n13002 Taddr:interpolation=even,place=city
|
||||
"""
|
||||
Then place contains exactly
|
||||
| object | class | type | address!dict |
|
||||
| N13001 | place | houses | 'interpolation': 'odd' |
|
||||
| N13002 | place | houses | 'interpolation': 'even' |
|
||||
|
||||
|
||||
Scenario: Footways
|
||||
When loading osm data
|
||||
"""
|
||||
|
||||
@@ -14,20 +14,24 @@ Feature: Updates of address interpolation objects
|
||||
n2 Taddr:housenumber=17
|
||||
w33 Thighway=residential,name=Tao Nn1,n2
|
||||
"""
|
||||
Then place contains
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W33 | highway | residential |
|
||||
|
||||
When updating osm data
|
||||
"""
|
||||
w99 Taddr:interpolation=odd Nn1,n2
|
||||
"""
|
||||
Then place contains
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W99 | place | houses |
|
||||
| W33 | highway | residential |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type |
|
||||
| 99 | odd |
|
||||
When indexing
|
||||
Then placex contains exactly
|
||||
| object | class | type |
|
||||
@@ -46,11 +50,13 @@ Feature: Updates of address interpolation objects
|
||||
n2 Taddr:housenumber=7
|
||||
w99 Taddr:interpolation=odd Nn1,n2
|
||||
"""
|
||||
Then place contains
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W99 | place | houses |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type |
|
||||
| 99 | odd |
|
||||
|
||||
When updating osm data
|
||||
"""
|
||||
@@ -60,6 +66,8 @@ Feature: Updates of address interpolation objects
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id |
|
||||
When indexing
|
||||
Then placex contains exactly
|
||||
| object | class | type |
|
||||
@@ -77,21 +85,27 @@ Feature: Updates of address interpolation objects
|
||||
w33 Thighway=residential Nn1,n2
|
||||
w99 Thighway=residential Nn1,n2
|
||||
"""
|
||||
Then place contains
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W33 | highway | residential |
|
||||
| W99 | highway | residential |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id |
|
||||
|
||||
When updating osm data
|
||||
"""
|
||||
w99 Taddr:interpolation=odd Nn1,n2
|
||||
"""
|
||||
Then place contains
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W99 | place | houses |
|
||||
| W33 | highway | residential |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type |
|
||||
| 99 | odd |
|
||||
When indexing
|
||||
Then placex contains exactly
|
||||
| object | class | type |
|
||||
@@ -110,11 +124,13 @@ Feature: Updates of address interpolation objects
|
||||
n2 Taddr:housenumber=17
|
||||
w99 Taddr:interpolation=odd Nn1,n2
|
||||
"""
|
||||
Then place contains
|
||||
Then place contains exactly
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W99 | place | houses |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type |
|
||||
| 99 | odd |
|
||||
|
||||
When updating osm data
|
||||
"""
|
||||
@@ -125,6 +141,8 @@ Feature: Updates of address interpolation objects
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W99 | highway | residential |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id |
|
||||
When indexing
|
||||
Then placex contains exactly
|
||||
| object | class | type |
|
||||
|
||||
@@ -112,7 +112,9 @@ Feature: Update of postcode only objects
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W34 | place | houses |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type |
|
||||
| 34 | odd |
|
||||
|
||||
When updating osm data
|
||||
"""
|
||||
@@ -122,9 +124,11 @@ Feature: Update of postcode only objects
|
||||
| object | class | type |
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
Then place_postcode contains exactly
|
||||
And place_postcode contains exactly
|
||||
| object | postcode |
|
||||
| W34 | 4456 |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id |
|
||||
When indexing
|
||||
Then location_property_osmline contains exactly
|
||||
| osm_id |
|
||||
@@ -158,7 +162,9 @@ Feature: Update of postcode only objects
|
||||
| N1 | place | house |
|
||||
| N2 | place | house |
|
||||
| W33 | highway | residential |
|
||||
| W34 | place | houses |
|
||||
And place_interpolation contains exactly
|
||||
| osm_id | type |
|
||||
| 34 | odd |
|
||||
And place_postcode contains exactly
|
||||
| object |
|
||||
When indexing
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Collector for BDD import acceptance tests.
|
||||
@@ -14,6 +14,7 @@ import re
|
||||
from collections import defaultdict
|
||||
|
||||
import psycopg
|
||||
import psycopg.sql as pysql
|
||||
|
||||
import pytest
|
||||
from pytest_bdd import when, then, given
|
||||
@@ -50,6 +51,34 @@ def _collect_place_ids(conn):
|
||||
return pids
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def row_factory(db_conn):
|
||||
def _insert_row(table, **data):
|
||||
columns = []
|
||||
placeholders = []
|
||||
values = []
|
||||
for k, v in data.items():
|
||||
columns.append(pysql.Identifier(k))
|
||||
if isinstance(v, tuple):
|
||||
placeholders.append(pysql.SQL(v[0]))
|
||||
values.append(v[1])
|
||||
elif isinstance(v, (pysql.Literal, pysql.SQL)):
|
||||
placeholders.append(v)
|
||||
else:
|
||||
placeholders.append(pysql.Placeholder())
|
||||
values.append(v)
|
||||
|
||||
sql = pysql.SQL("INSERT INTO {table} ({columns}) VALUES({values})")\
|
||||
.format(table=pysql.Identifier(table),
|
||||
columns=pysql.SQL(',').join(columns),
|
||||
values=pysql.SQL(',').join(placeholders))
|
||||
|
||||
db_conn.execute(sql, values)
|
||||
db_conn.commit()
|
||||
|
||||
return _insert_row
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_config_env(pytestconfig):
|
||||
dbname = pytestconfig.getini('nominatim_test_db')
|
||||
@@ -85,18 +114,36 @@ def import_places(db_conn, named, datatable, node_grid):
|
||||
|
||||
|
||||
@given(step_parse('the entrances'), target_fixture=None)
|
||||
def import_place_entrances(db_conn, datatable, node_grid):
|
||||
def import_place_entrances(row_factory, datatable, node_grid):
|
||||
""" Insert todo rows into the place_entrance table.
|
||||
"""
|
||||
with db_conn.cursor() as cur:
|
||||
for row in datatable[1:]:
|
||||
data = PlaceColumn(node_grid).add_row(datatable[0], row, False)
|
||||
assert data.columns['osm_type'] == 'N'
|
||||
for row in datatable[1:]:
|
||||
data = PlaceColumn(node_grid).add_row(datatable[0], row, False)
|
||||
assert data.columns['osm_type'] == 'N'
|
||||
|
||||
cur.execute("""INSERT INTO place_entrance (osm_id, type, extratags, geometry)
|
||||
VALUES (%s, %s, %s, {})""".format(data.get_wkt()),
|
||||
(data.columns['osm_id'], data.columns['type'],
|
||||
data.columns.get('extratags')))
|
||||
params = {'osm_id': data.columns['osm_id'],
|
||||
'type': data.columns['type'],
|
||||
'extratags': data.columns.get('extratags'),
|
||||
'geometry': pysql.SQL(data.get_wkt())}
|
||||
|
||||
row_factory('place_entrance', **params)
|
||||
|
||||
|
||||
@given(step_parse('the interpolations'), target_fixture=None)
|
||||
def import_place_interpolations(row_factory, datatable, node_grid):
|
||||
""" Insert todo rows into the place_entrance table.
|
||||
"""
|
||||
for row in datatable[1:]:
|
||||
data = PlaceColumn(node_grid).add_row(datatable[0], row, False)
|
||||
assert data.columns['osm_type'] == 'W'
|
||||
|
||||
params = {'osm_id': data.columns['osm_id'],
|
||||
'type': data.columns['type'],
|
||||
'address': data.columns.get('address'),
|
||||
'nodes': [int(x) for x in data.columns['nodes'].split(',')],
|
||||
'geometry': pysql.SQL(data.get_wkt())}
|
||||
|
||||
row_factory('place_interpolation', **params)
|
||||
|
||||
|
||||
@given(step_parse('the postcodes'), target_fixture=None)
|
||||
@@ -135,43 +182,41 @@ def import_place_postcode(db_conn, datatable, node_grid):
|
||||
|
||||
|
||||
@given('the ways', target_fixture=None)
|
||||
def import_ways(db_conn, datatable):
|
||||
def import_ways(row_factory, datatable):
|
||||
""" Import raw ways into the osm2pgsql way middle table.
|
||||
"""
|
||||
with db_conn.cursor() as cur:
|
||||
id_idx = datatable[0].index('id')
|
||||
node_idx = datatable[0].index('nodes')
|
||||
for line in datatable[1:]:
|
||||
tags = psycopg.types.json.Json(
|
||||
{k[5:]: v for k, v in zip(datatable[0], line)
|
||||
if k.startswith("tags+")})
|
||||
nodes = [int(x) for x in line[node_idx].split(',')]
|
||||
|
||||
cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)",
|
||||
(line[id_idx], nodes, tags))
|
||||
id_idx = datatable[0].index('id')
|
||||
node_idx = datatable[0].index('nodes')
|
||||
for line in datatable[1:]:
|
||||
row_factory('planet_osm_ways',
|
||||
id=line[id_idx],
|
||||
nodes=[int(x) for x in line[node_idx].split(',')],
|
||||
tags=psycopg.types.json.Json(
|
||||
{k[5:]: v for k, v in zip(datatable[0], line)
|
||||
if k.startswith("tags+")}))
|
||||
|
||||
|
||||
@given('the relations', target_fixture=None)
|
||||
def import_rels(db_conn, datatable):
|
||||
def import_rels(row_factory, datatable):
|
||||
""" Import raw relations into the osm2pgsql relation middle table.
|
||||
"""
|
||||
with db_conn.cursor() as cur:
|
||||
id_idx = datatable[0].index('id')
|
||||
memb_idx = datatable[0].index('members')
|
||||
for line in datatable[1:]:
|
||||
tags = psycopg.types.json.Json(
|
||||
{k[5:]: v for k, v in zip(datatable[0], line)
|
||||
if k.startswith("tags+")})
|
||||
members = []
|
||||
if line[memb_idx]:
|
||||
for member in line[memb_idx].split(','):
|
||||
m = re.fullmatch(r'\s*([RWN])(\d+)(?::(\S+))?\s*', member)
|
||||
if not m:
|
||||
raise ValueError(f'Illegal member {member}.')
|
||||
members.append({'ref': int(m[2]), 'role': m[3] or '', 'type': m[1]})
|
||||
id_idx = datatable[0].index('id')
|
||||
memb_idx = datatable[0].index('members')
|
||||
for line in datatable[1:]:
|
||||
tags = psycopg.types.json.Json(
|
||||
{k[5:]: v for k, v in zip(datatable[0], line)
|
||||
if k.startswith("tags+")})
|
||||
members = []
|
||||
if line[memb_idx]:
|
||||
for member in line[memb_idx].split(','):
|
||||
m = re.fullmatch(r'\s*([RWN])(\d+)(?::(\S+))?\s*', member)
|
||||
if not m:
|
||||
raise ValueError(f'Illegal member {member}.')
|
||||
members.append({'ref': int(m[2]), 'role': m[3] or '', 'type': m[1]})
|
||||
|
||||
cur.execute('INSERT INTO planet_osm_rels (id, tags, members) VALUES (%s, %s, %s)',
|
||||
(int(line[id_idx]), tags, psycopg.types.json.Json(members)))
|
||||
row_factory('planet_osm_rels',
|
||||
id=int(line[id_idx]), tags=tags,
|
||||
members=psycopg.types.json.Json(members))
|
||||
|
||||
|
||||
@when('importing', target_fixture='place_ids')
|
||||
@@ -221,6 +266,28 @@ def update_place_entrances(db_conn, datatable, node_grid):
|
||||
db_conn.commit()
|
||||
|
||||
|
||||
@when('updating interpolations', target_fixture=None)
|
||||
def update_place_interpolations(db_conn, row_factory, update_config, datatable, node_grid):
|
||||
""" Update rows in the place_entrance table.
|
||||
"""
|
||||
for row in datatable[1:]:
|
||||
data = PlaceColumn(node_grid).add_row(datatable[0], row, False)
|
||||
assert data.columns['osm_type'] == 'W'
|
||||
|
||||
params = {'osm_id': data.columns['osm_id'],
|
||||
'type': data.columns['type'],
|
||||
'address': data.columns.get('address'),
|
||||
'nodes': [int(x) for x in data.columns['nodes'].split(',')],
|
||||
'geometry': pysql.SQL(data.get_wkt())}
|
||||
|
||||
row_factory('place_interpolation', **params)
|
||||
|
||||
db_conn.execute('SELECT flush_deleted_places()')
|
||||
db_conn.commit()
|
||||
|
||||
cli.nominatim(['index', '-q', '--minrank', '30'], update_config.environ)
|
||||
|
||||
|
||||
@when('refreshing postcodes')
|
||||
def do_postcode_update(update_config):
|
||||
""" Recompute the postcode centroids.
|
||||
@@ -237,6 +304,8 @@ def do_delete_place(db_conn, update_config, node_grid, otype, oid):
|
||||
cur.execute('TRUNCATE place_to_be_deleted')
|
||||
cur.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
|
||||
(otype, oid))
|
||||
cur.execute('DELETE FROM place_interpolation WHERE osm_id = %s',
|
||||
(oid, ))
|
||||
cur.execute('SELECT flush_deleted_places()')
|
||||
if otype == 'N':
|
||||
cur.execute('DELETE FROM place_entrance WHERE osm_id = %s',
|
||||
|
||||
@@ -43,7 +43,7 @@ def opl_writer(tmp_path, node_grid):
|
||||
def _write(data):
|
||||
fname = tmp_path / f"test_osm_{nr[0]}.opl"
|
||||
nr[0] += 1
|
||||
with fname.open('wt') as fd:
|
||||
with fname.open('wt', encoding='utf-8') as fd:
|
||||
for line in data.split('\n'):
|
||||
if line.startswith('n') and ' x' not in line:
|
||||
coord = node_grid.get(line[1:].split(' ')[0]) \
|
||||
@@ -59,7 +59,7 @@ def opl_writer(tmp_path, node_grid):
|
||||
@given('the lua style file', target_fixture='osm2pgsql_options')
|
||||
def set_lua_style_file(osm2pgsql_options, docstring, tmp_path):
|
||||
style = tmp_path / 'custom.lua'
|
||||
style.write_text(docstring)
|
||||
style.write_text(docstring, encoding='utf-8')
|
||||
osm2pgsql_options['osm2pgsql_style'] = str(style)
|
||||
|
||||
return osm2pgsql_options
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Various helper classes for running Nominatim commands.
|
||||
@@ -54,15 +54,14 @@ class APIRunner:
|
||||
def create_engine_starlette(self, environ):
|
||||
import nominatim_api.server.starlette.server
|
||||
from asgi_lifespan import LifespanManager
|
||||
import httpx
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
async def _request(endpoint, params, http_headers):
|
||||
app = nominatim_api.server.starlette.server.get_application(None, environ)
|
||||
|
||||
async with LifespanManager(app):
|
||||
async with httpx.AsyncClient(app=app, base_url="http://nominatim.test") as client:
|
||||
response = await client.get("/" + endpoint, params=params,
|
||||
headers=http_headers)
|
||||
client = TestClient(app, base_url="http://nominatim.test")
|
||||
response = client.get("/" + endpoint, params=params, headers=http_headers)
|
||||
|
||||
return APIResponse(endpoint, response.status_code,
|
||||
response.text, response.headers)
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Helper functions to compare expected values.
|
||||
"""
|
||||
import ast
|
||||
import collections.abc
|
||||
import json
|
||||
import re
|
||||
@@ -58,7 +59,10 @@ COMPARISON_FUNCS = {
|
||||
None: lambda val, exp: str(val) == exp,
|
||||
'i': lambda val, exp: str(val).lower() == exp.lower(),
|
||||
'fm': lambda val, exp: re.fullmatch(exp, val) is not None,
|
||||
'dict': lambda val, exp: val is None if exp == '-' else (val == eval('{' + exp + '}')),
|
||||
'dict': lambda val, exp: (val is None if exp == '-'
|
||||
else (val == ast.literal_eval('{' + exp + '}'))),
|
||||
'ints': lambda val, exp: (val is None if exp == '-'
|
||||
else (val == [int(i) for i in exp.split(',')])),
|
||||
'in_box': within_box
|
||||
}
|
||||
|
||||
@@ -82,6 +86,8 @@ class ResultAttr:
|
||||
!fm - consider comparison string a regular expression and match full value
|
||||
!wkt - convert the expected value to a WKT string before comparing
|
||||
!in_box - the expected value is a comma-separated bbox description
|
||||
!dict - compare as a dictitionary, member order does not matter
|
||||
!ints - compare as integer array
|
||||
"""
|
||||
|
||||
def __init__(self, obj, key, grid=None):
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Helper classes for filling the place table.
|
||||
"""
|
||||
import ast
|
||||
import random
|
||||
import string
|
||||
|
||||
@@ -35,7 +36,8 @@ class PlaceColumn:
|
||||
self._add_hstore(
|
||||
'name',
|
||||
'name',
|
||||
''.join(random.choices(string.printable, k=random.randrange(30))),
|
||||
''.join(random.choices(string.ascii_uppercase)
|
||||
+ random.choices(string.printable, k=random.randrange(30))),
|
||||
)
|
||||
|
||||
return self
|
||||
@@ -50,9 +52,8 @@ class PlaceColumn:
|
||||
elif key.startswith('addr+'):
|
||||
self._add_hstore('address', key[5:], value)
|
||||
elif key in ('name', 'address', 'extratags'):
|
||||
self.columns[key] = eval('{' + value + '}')
|
||||
self.columns[key] = ast.literal_eval('{' + value + '}')
|
||||
else:
|
||||
assert key in ('class', 'type'), "Unknown column '{}'.".format(key)
|
||||
self.columns[key] = None if value == '' else value
|
||||
|
||||
def _set_key_name(self, value):
|
||||
|
||||
@@ -58,7 +58,7 @@ gb:
|
||||
pattern: "(l?ld[A-Z0-9]?) ?(dll)"
|
||||
output: \1 \2
|
||||
|
||||
""")
|
||||
""", encoding='utf-8')
|
||||
|
||||
return project_env
|
||||
|
||||
|
||||
@@ -91,8 +91,9 @@ class TestCliWithDb:
|
||||
postcode_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer,
|
||||
'index_postcodes')
|
||||
|
||||
has_pending_retvals = [True, False]
|
||||
monkeypatch.setattr(nominatim_db.indexer.indexer.Indexer, 'has_pending',
|
||||
[False, True].pop)
|
||||
lambda *args, **kwargs: has_pending_retvals.pop(0))
|
||||
|
||||
assert self.call_nominatim('index', *params) == 0
|
||||
|
||||
|
||||
42
test/python/cli/test_cmd_index.py
Normal file
42
test/python/cli/test_cmd_index.py
Normal file
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Tests for index command of the command-line interface wrapper.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
import nominatim_db.indexer.indexer
|
||||
|
||||
|
||||
class TestCliIndexWithDb:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cli_call(self, cli_call, cli_tokenizer_mock):
|
||||
self.call_nominatim = cli_call
|
||||
self.tokenizer_mock = cli_tokenizer_mock
|
||||
|
||||
def test_index_empty_subset(self, monkeypatch, async_mock_func_factory, placex_row):
|
||||
placex_row(rank_address=1, indexed_status=1)
|
||||
placex_row(rank_address=20, indexed_status=1)
|
||||
|
||||
mocks = [
|
||||
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_boundaries'),
|
||||
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_by_rank'),
|
||||
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_postcodes'),
|
||||
]
|
||||
|
||||
def _reject_repeat_call(*args, **kwargs):
|
||||
assert False, "Did not expect multiple Indexer.has_pending invocations"
|
||||
|
||||
has_pending_calls = [nominatim_db.indexer.indexer.Indexer.has_pending, _reject_repeat_call]
|
||||
monkeypatch.setattr(nominatim_db.indexer.indexer.Indexer, 'has_pending',
|
||||
lambda *args, **kwargs: has_pending_calls.pop(0)(*args, **kwargs))
|
||||
|
||||
assert self.call_nominatim('index', '--minrank', '5', '--maxrank', '10') == 0
|
||||
|
||||
for mock in mocks:
|
||||
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Test for loading dotenv configuration.
|
||||
@@ -48,7 +48,7 @@ def test_no_project_dir(make_config):
|
||||
@pytest.mark.parametrize("val", ('apache', '"apache"'))
|
||||
def test_prefer_project_setting_over_default(make_config, val, tmp_path):
|
||||
envfile = tmp_path / '.env'
|
||||
envfile.write_text('NOMINATIM_DATABASE_WEBUSER={}\n'.format(val))
|
||||
envfile.write_text('NOMINATIM_DATABASE_WEBUSER={}\n'.format(val), encoding='utf-8')
|
||||
|
||||
config = make_config(tmp_path)
|
||||
|
||||
@@ -57,7 +57,7 @@ def test_prefer_project_setting_over_default(make_config, val, tmp_path):
|
||||
|
||||
def test_prefer_os_environ_over_project_setting(make_config, monkeypatch, tmp_path):
|
||||
envfile = tmp_path / '.env'
|
||||
envfile.write_text('NOMINATIM_DATABASE_WEBUSER=apache\n')
|
||||
envfile.write_text('NOMINATIM_DATABASE_WEBUSER=apache\n', encoding='utf-8')
|
||||
|
||||
monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'nobody')
|
||||
|
||||
@@ -68,13 +68,13 @@ def test_prefer_os_environ_over_project_setting(make_config, monkeypatch, tmp_pa
|
||||
|
||||
def test_prefer_os_environ_can_unset_project_setting(make_config, monkeypatch, tmp_path):
|
||||
envfile = tmp_path / '.env'
|
||||
envfile.write_text('NOMINATIM_DATABASE_WEBUSER=apache\n')
|
||||
envfile.write_text('NOMINATIM_OSM2PGSQL_BINARY=osm2pgsql\n', encoding='utf-8')
|
||||
|
||||
monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', '')
|
||||
monkeypatch.setenv('NOMINATIM_OSM2PGSQL_BINARY', '')
|
||||
|
||||
config = make_config(tmp_path)
|
||||
|
||||
assert config.DATABASE_WEBUSER == ''
|
||||
assert config.OSM2PGSQL_BINARY == ''
|
||||
|
||||
|
||||
def test_get_os_env_add_defaults(make_config, monkeypatch):
|
||||
@@ -200,14 +200,15 @@ def test_get_path_empty(make_config):
|
||||
assert not config.get_path('TOKENIZER_CONFIG')
|
||||
|
||||
|
||||
def test_get_path_absolute(make_config, monkeypatch):
|
||||
def test_get_path_absolute(make_config, monkeypatch, tmp_path):
|
||||
config = make_config()
|
||||
|
||||
monkeypatch.setenv('NOMINATIM_FOOBAR', '/dont/care')
|
||||
p = (tmp_path / "does_not_exist").resolve()
|
||||
monkeypatch.setenv('NOMINATIM_FOOBAR', str(p))
|
||||
result = config.get_path('FOOBAR')
|
||||
|
||||
assert isinstance(result, Path)
|
||||
assert str(result) == '/dont/care'
|
||||
assert str(result) == str(p)
|
||||
|
||||
|
||||
def test_get_path_relative(make_config, monkeypatch, tmp_path):
|
||||
@@ -232,7 +233,7 @@ def test_get_import_style_intern(make_config, src_dir, monkeypatch):
|
||||
|
||||
def test_get_import_style_extern_relative(make_config_path, monkeypatch):
|
||||
config = make_config_path()
|
||||
(config.project_dir / 'custom.style').write_text('x')
|
||||
(config.project_dir / 'custom.style').write_text('x', encoding='utf-8')
|
||||
|
||||
monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', 'custom.style')
|
||||
|
||||
@@ -243,7 +244,7 @@ def test_get_import_style_extern_absolute(make_config, tmp_path, monkeypatch):
|
||||
config = make_config()
|
||||
cfgfile = tmp_path / 'test.style'
|
||||
|
||||
cfgfile.write_text('x')
|
||||
cfgfile.write_text('x', encoding='utf-8')
|
||||
|
||||
monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', str(cfgfile))
|
||||
|
||||
@@ -254,10 +255,10 @@ def test_load_subconf_from_project_dir(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.project_dir / 'test.yaml'
|
||||
testfile.write_text('cow: muh\ncat: miau\n')
|
||||
testfile.write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('cow: miau\ncat: muh\n')
|
||||
testfile.write_text('cow: miau\ncat: muh\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml')
|
||||
|
||||
@@ -268,7 +269,7 @@ def test_load_subconf_from_settings_dir(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('cow: muh\ncat: miau\n')
|
||||
testfile.write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml')
|
||||
|
||||
@@ -280,7 +281,7 @@ def test_load_subconf_empty_env_conf(make_config_path, monkeypatch):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('cow: muh\ncat: miau\n')
|
||||
testfile.write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml', config='MY_CONFIG')
|
||||
|
||||
@@ -291,8 +292,8 @@ def test_load_subconf_env_absolute_found(make_config_path, monkeypatch, tmp_path
|
||||
monkeypatch.setenv('NOMINATIM_MY_CONFIG', str(tmp_path / 'other.yaml'))
|
||||
config = make_config_path()
|
||||
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n')
|
||||
(tmp_path / 'other.yaml').write_text('dog: muh\nfrog: miau\n')
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
(tmp_path / 'other.yaml').write_text('dog: muh\nfrog: miau\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml', config='MY_CONFIG')
|
||||
|
||||
@@ -303,7 +304,7 @@ def test_load_subconf_env_absolute_not_found(make_config_path, monkeypatch, tmp_
|
||||
monkeypatch.setenv('NOMINATIM_MY_CONFIG', str(tmp_path / 'other.yaml'))
|
||||
config = make_config_path()
|
||||
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n')
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
|
||||
with pytest.raises(UsageError, match='Config file not found.'):
|
||||
config.load_sub_configuration('test.yaml', config='MY_CONFIG')
|
||||
@@ -314,8 +315,8 @@ def test_load_subconf_env_relative_found(make_config_path, monkeypatch, location
|
||||
monkeypatch.setenv('NOMINATIM_MY_CONFIG', 'other.yaml')
|
||||
config = make_config_path()
|
||||
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n')
|
||||
(getattr(config, location) / 'other.yaml').write_text('dog: bark\n')
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
(getattr(config, location) / 'other.yaml').write_text('dog: bark\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml', config='MY_CONFIG')
|
||||
|
||||
@@ -326,7 +327,7 @@ def test_load_subconf_env_relative_not_found(make_config_path, monkeypatch):
|
||||
monkeypatch.setenv('NOMINATIM_MY_CONFIG', 'other.yaml')
|
||||
config = make_config_path()
|
||||
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n')
|
||||
(config.config_dir / 'test.yaml').write_text('cow: muh\ncat: miau\n', encoding='utf-8')
|
||||
|
||||
with pytest.raises(UsageError, match='Config file not found.'):
|
||||
config.load_sub_configuration('test.yaml', config='MY_CONFIG')
|
||||
@@ -335,7 +336,7 @@ def test_load_subconf_env_relative_not_found(make_config_path, monkeypatch):
|
||||
def test_load_subconf_json(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
(config.project_dir / 'test.json').write_text('{"cow": "muh", "cat": "miau"}')
|
||||
(config.project_dir / 'test.json').write_text('{"cow": "muh", "cat": "miau"}', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.json')
|
||||
|
||||
@@ -352,7 +353,7 @@ def test_load_subconf_not_found(make_config_path):
|
||||
def test_load_subconf_env_unknown_format(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
(config.project_dir / 'test.xml').write_text('<html></html>')
|
||||
(config.project_dir / 'test.xml').write_text('<html></html>', encoding='utf-8')
|
||||
|
||||
with pytest.raises(UsageError, match='unknown format'):
|
||||
config.load_sub_configuration('test.xml')
|
||||
@@ -362,8 +363,8 @@ def test_load_subconf_include_absolute(make_config_path, tmp_path):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text(f'base: !include {tmp_path}/inc.yaml\n')
|
||||
(tmp_path / 'inc.yaml').write_text('first: 1\nsecond: 2\n')
|
||||
testfile.write_text(f'base: !include {tmp_path}/inc.yaml\n', encoding='utf-8')
|
||||
(tmp_path / 'inc.yaml').write_text('first: 1\nsecond: 2\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml')
|
||||
|
||||
@@ -375,8 +376,8 @@ def test_load_subconf_include_relative(make_config_path, tmp_path, location):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('base: !include inc.yaml\n')
|
||||
(getattr(config, location) / 'inc.yaml').write_text('first: 1\nsecond: 2\n')
|
||||
testfile.write_text('base: !include inc.yaml\n', encoding='utf-8')
|
||||
(getattr(config, location) / 'inc.yaml').write_text('first: 1\nsecond: 2\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml')
|
||||
|
||||
@@ -387,8 +388,8 @@ def test_load_subconf_include_bad_format(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('base: !include inc.txt\n')
|
||||
(config.config_dir / 'inc.txt').write_text('first: 1\nsecond: 2\n')
|
||||
testfile.write_text('base: !include inc.txt\n', encoding='utf-8')
|
||||
(config.config_dir / 'inc.txt').write_text('first: 1\nsecond: 2\n', encoding='utf-8')
|
||||
|
||||
with pytest.raises(UsageError, match='Cannot handle config file format.'):
|
||||
config.load_sub_configuration('test.yaml')
|
||||
@@ -398,7 +399,7 @@ def test_load_subconf_include_not_found(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('base: !include inc.txt\n')
|
||||
testfile.write_text('base: !include inc.txt\n', encoding='utf-8')
|
||||
|
||||
with pytest.raises(UsageError, match='Config file not found.'):
|
||||
config.load_sub_configuration('test.yaml')
|
||||
@@ -408,9 +409,9 @@ def test_load_subconf_include_recursive(make_config_path):
|
||||
config = make_config_path()
|
||||
|
||||
testfile = config.config_dir / 'test.yaml'
|
||||
testfile.write_text('base: !include inc.yaml\n')
|
||||
(config.config_dir / 'inc.yaml').write_text('- !include more.yaml\n- upper\n')
|
||||
(config.config_dir / 'more.yaml').write_text('- the end\n')
|
||||
testfile.write_text('base: !include inc.yaml\n', encoding='utf-8')
|
||||
(config.config_dir / 'inc.yaml').write_text('- !include more.yaml\n- upper\n', encoding='utf-8')
|
||||
(config.config_dir / 'more.yaml').write_text('- the end\n', encoding='utf-8')
|
||||
|
||||
rules = config.load_sub_configuration('test.yaml')
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ def test_load_default_module_with_hyphen(test_config):
|
||||
def test_load_plugin_module(test_config, tmp_path):
|
||||
(tmp_path / 'project' / 'testpath').mkdir()
|
||||
(tmp_path / 'project' / 'testpath' / 'mymod.py')\
|
||||
.write_text("def my_test_function():\n return 'gjwitlsSG42TG%'")
|
||||
.write_text("def my_test_function():\n return 'gjwitlsSG42TG%'", encoding='utf-8')
|
||||
|
||||
module = test_config.load_plugin_module('testpath/mymod.py', 'private.something')
|
||||
|
||||
@@ -49,7 +49,7 @@ def test_load_plugin_module(test_config, tmp_path):
|
||||
|
||||
# also test reloading module
|
||||
(tmp_path / 'project' / 'testpath' / 'mymod.py')\
|
||||
.write_text("def my_test_function():\n return 'hjothjorhj'")
|
||||
.write_text("def my_test_function():\n return 'hjothjorhj'", encoding='utf-8')
|
||||
|
||||
module = test_config.load_plugin_module('testpath/mymod.py', 'private.something')
|
||||
|
||||
@@ -61,9 +61,9 @@ def test_load_external_library_module(test_config, tmp_path, monkeypatch):
|
||||
pythonpath = tmp_path / 'priv-python'
|
||||
pythonpath.mkdir()
|
||||
(pythonpath / MODULE_NAME).mkdir()
|
||||
(pythonpath / MODULE_NAME / '__init__.py').write_text('')
|
||||
(pythonpath / MODULE_NAME / '__init__.py').write_text('', encoding='utf-8')
|
||||
(pythonpath / MODULE_NAME / 'tester.py')\
|
||||
.write_text("def my_test_function():\n return 'gjwitlsSG42TG%'")
|
||||
.write_text("def my_test_function():\n return 'gjwitlsSG42TG%'", encoding='utf-8')
|
||||
|
||||
monkeypatch.syspath_prepend(pythonpath)
|
||||
|
||||
@@ -73,7 +73,7 @@ def test_load_external_library_module(test_config, tmp_path, monkeypatch):
|
||||
|
||||
# also test reloading module
|
||||
(pythonpath / MODULE_NAME / 'tester.py')\
|
||||
.write_text("def my_test_function():\n return 'dfigjreigj'")
|
||||
.write_text("def my_test_function():\n return 'dfigjreigj'", encoding='utf-8')
|
||||
|
||||
module = test_config.load_plugin_module(f'{MODULE_NAME}.tester', 'private.something')
|
||||
|
||||
|
||||
@@ -2,12 +2,16 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
import itertools
|
||||
import sys
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
if sys.platform == 'win32':
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql as pysql
|
||||
import pytest
|
||||
@@ -17,12 +21,11 @@ SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
|
||||
sys.path.insert(0, str(SRC_DIR / 'src'))
|
||||
|
||||
from nominatim_db.config import Configuration
|
||||
from nominatim_db.db import connection
|
||||
from nominatim_db.db import connection, properties
|
||||
from nominatim_db.db.sql_preprocessor import SQLPreprocessor
|
||||
import nominatim_db.tokenizer.factory
|
||||
|
||||
import dummy_tokenizer
|
||||
import mocks
|
||||
from cursor import CursorForTesting
|
||||
|
||||
|
||||
@@ -60,7 +63,7 @@ def temp_db(monkeypatch):
|
||||
|
||||
with psycopg.connect(dbname='postgres', autocommit=True) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
|
||||
cur.execute(pysql.SQL('DROP DATABASE IF EXISTS') + pysql.Identifier(name))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -104,7 +107,9 @@ def table_factory(temp_db_conn):
|
||||
"""
|
||||
def mk_table(name, definition='id INT', content=None):
|
||||
with psycopg.ClientCursor(temp_db_conn) as cur:
|
||||
cur.execute('CREATE TABLE {} ({})'.format(name, definition))
|
||||
cur.execute(pysql.SQL("CREATE TABLE {} ({})")
|
||||
.format(pysql.Identifier(name),
|
||||
pysql.SQL(definition)))
|
||||
if content:
|
||||
sql = pysql.SQL("INSERT INTO {} VALUES ({})")\
|
||||
.format(pysql.Identifier(name),
|
||||
@@ -130,28 +135,50 @@ def project_env(tmp_path):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def property_table(table_factory, temp_db_conn):
|
||||
table_factory('nominatim_properties', 'property TEXT, value TEXT')
|
||||
|
||||
return mocks.MockPropertyTable(temp_db_conn)
|
||||
def country_table(table_factory):
|
||||
table_factory('country_name', 'partition INT, country_code varchar(2), name hstore')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def status_table(table_factory):
|
||||
def country_row(country_table, temp_db_cursor):
|
||||
def _add(partition=None, country=None, names=None):
|
||||
temp_db_cursor.insert_row('country_name', partition=partition,
|
||||
country_code=country, name=names)
|
||||
|
||||
return _add
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def load_sql(temp_db_conn, country_table):
|
||||
conf = Configuration(None)
|
||||
|
||||
def _run(*filename, **kwargs):
|
||||
for fn in filename:
|
||||
SQLPreprocessor(temp_db_conn, conf).run_sql_file(temp_db_conn, fn, **kwargs)
|
||||
|
||||
return _run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def property_table(load_sql, temp_db_conn):
|
||||
load_sql('tables/nominatim_properties.sql')
|
||||
|
||||
class _PropTable:
|
||||
def set(self, name, value):
|
||||
properties.set_property(temp_db_conn, name, value)
|
||||
|
||||
def get(self, name):
|
||||
return properties.get_property(temp_db_conn, name)
|
||||
|
||||
return _PropTable()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def status_table(load_sql):
|
||||
""" Create an empty version of the status table and
|
||||
the status logging table.
|
||||
"""
|
||||
table_factory('import_status',
|
||||
"""lastimportdate timestamp with time zone NOT NULL,
|
||||
sequence_id integer,
|
||||
indexed boolean""")
|
||||
table_factory('import_osmosis_log',
|
||||
"""batchend timestamp,
|
||||
batchseq integer,
|
||||
batchsize bigint,
|
||||
starttime timestamp,
|
||||
endtime timestamp,
|
||||
event text""")
|
||||
load_sql('tables/status.sql')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -165,9 +192,9 @@ def place_table(temp_db_with_extensions, table_factory):
|
||||
type text NOT NULL,
|
||||
name hstore,
|
||||
admin_level smallint,
|
||||
address hstore,
|
||||
extratags hstore,
|
||||
geometry Geometry(Geometry,4326) NOT NULL""")
|
||||
address HSTORE,
|
||||
extratags HSTORE,
|
||||
geometry GEOMETRY(Geometry,4326) NOT NULL""")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -176,12 +203,14 @@ def place_row(place_table, temp_db_cursor):
|
||||
prerequisite to the fixture.
|
||||
"""
|
||||
idseq = itertools.count(1001)
|
||||
|
||||
def _insert(osm_type='N', osm_id=None, cls='amenity', typ='cafe', names=None,
|
||||
admin_level=None, address=None, extratags=None, geom=None):
|
||||
temp_db_cursor.execute("INSERT INTO place VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
||||
(osm_id or next(idseq), osm_type, cls, typ, names,
|
||||
admin_level, address, extratags,
|
||||
geom or 'SRID=4326;POINT(0 0)'))
|
||||
admin_level=None, address=None, extratags=None, geom='POINT(0 0)'):
|
||||
args = {'osm_type': osm_type, 'osm_id': osm_id or next(idseq),
|
||||
'class': cls, 'type': typ, 'name': names, 'admin_level': admin_level,
|
||||
'address': address, 'extratags': extratags,
|
||||
'geometry': _with_srid(geom)}
|
||||
temp_db_cursor.insert_row('place', **args)
|
||||
|
||||
return _insert
|
||||
|
||||
@@ -194,57 +223,140 @@ def place_postcode_table(temp_db_with_extensions, table_factory):
|
||||
"""osm_type char(1) NOT NULL,
|
||||
osm_id bigint NOT NULL,
|
||||
postcode text NOT NULL,
|
||||
country_code text,
|
||||
centroid Geometry(Point, 4326) NOT NULL,
|
||||
geometry Geometry(Geometry, 4326)""")
|
||||
country_code TEXT,
|
||||
centroid GEOMETRY(Point, 4326) NOT NULL,
|
||||
geometry GEOMETRY(Geometry, 4326)""")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def place_postcode_row(place_postcode_table, temp_db_cursor):
|
||||
""" A factory for rows in the place table. The table is created as a
|
||||
""" A factory for rows in the place_postcode table. The table is created as a
|
||||
prerequisite to the fixture.
|
||||
"""
|
||||
idseq = itertools.count(5001)
|
||||
|
||||
def _insert(osm_type='N', osm_id=None, postcode=None, country=None,
|
||||
centroid=None, geom=None):
|
||||
temp_db_cursor.execute("INSERT INTO place_postcode VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(osm_type, osm_id or next(idseq),
|
||||
postcode, country,
|
||||
_with_srid(centroid, 'POINT(12.0 4.0)'),
|
||||
_with_srid(geom)))
|
||||
centroid='POINT(12.0 4.0)', geom=None):
|
||||
temp_db_cursor.insert_row('place_postcode',
|
||||
osm_type=osm_type, osm_id=osm_id or next(idseq),
|
||||
postcode=postcode, country_code=country,
|
||||
centroid=_with_srid(centroid),
|
||||
geometry=_with_srid(geom))
|
||||
|
||||
return _insert
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def placex_table(temp_db_with_extensions, temp_db_conn):
|
||||
""" Create an empty version of the place table.
|
||||
def place_interpolation_table(temp_db_with_extensions, table_factory):
|
||||
""" Create an empty version of the place_interpolation table.
|
||||
"""
|
||||
return mocks.MockPlacexTable(temp_db_conn)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def osmline_table(temp_db_with_extensions, table_factory):
|
||||
table_factory('location_property_osmline',
|
||||
"""place_id BIGINT,
|
||||
osm_id BIGINT,
|
||||
parent_place_id BIGINT,
|
||||
geometry_sector INTEGER,
|
||||
indexed_date TIMESTAMP,
|
||||
startnumber INTEGER,
|
||||
endnumber INTEGER,
|
||||
partition SMALLINT,
|
||||
indexed_status SMALLINT,
|
||||
linegeo GEOMETRY,
|
||||
interpolationtype TEXT,
|
||||
table_factory('place_interpolation',
|
||||
"""osm_id bigint NOT NULL,
|
||||
type TEXT,
|
||||
address HSTORE,
|
||||
postcode TEXT,
|
||||
country_code VARCHAR(2)""")
|
||||
nodes BIGINT[],
|
||||
geometry GEOMETRY(Geometry, 4326)""")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sql_preprocessor_cfg(tmp_path, table_factory, temp_db_with_extensions):
|
||||
table_factory('country_name', 'partition INT', ((0, ), (1, ), (2, )))
|
||||
def place_interpolation_row(place_interpolation_table, temp_db_cursor):
|
||||
""" A factory for rows in the place_interpolation table. The table is created as a
|
||||
prerequisite to the fixture.
|
||||
"""
|
||||
idseq = itertools.count(30001)
|
||||
|
||||
def _insert(osm_id=None, typ='odd', address=None,
|
||||
nodes=None, geom='LINESTRING(0.1 0.21, 0.1 0.2)'):
|
||||
params = {'osm_id': osm_id or next(idseq),
|
||||
'type': typ, 'address': address, 'nodes': nodes,
|
||||
'geometry': _with_srid(geom)}
|
||||
temp_db_cursor.insert_row('place_interpolation', **params)
|
||||
|
||||
return _insert
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def placex_table(temp_db_with_extensions, temp_db_conn, load_sql, place_table):
|
||||
""" Create an empty version of the placex table.
|
||||
"""
|
||||
load_sql('tables/placex.sql')
|
||||
temp_db_conn.execute("CREATE SEQUENCE IF NOT EXISTS seq_place START 1")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def placex_row(placex_table, temp_db_cursor):
|
||||
""" A factory for rows in the placex table. The table is created as a
|
||||
prerequisite to the fixture.
|
||||
"""
|
||||
idseq = itertools.count(1001)
|
||||
|
||||
def _add(osm_type='N', osm_id=None, cls='amenity', typ='cafe', names=None,
|
||||
admin_level=None, address=None, extratags=None, geom='POINT(10 4)',
|
||||
country=None, housenumber=None, rank_search=30, rank_address=30,
|
||||
centroid='POINT(10 4)', indexed_status=0, indexed_date=None):
|
||||
args = {'place_id': pysql.SQL("nextval('seq_place')"),
|
||||
'osm_type': osm_type, 'osm_id': osm_id or next(idseq),
|
||||
'class': cls, 'type': typ, 'name': names, 'admin_level': admin_level,
|
||||
'address': address, 'housenumber': housenumber,
|
||||
'rank_search': rank_search, 'rank_address': rank_address,
|
||||
'extratags': extratags,
|
||||
'centroid': _with_srid(centroid), 'geometry': _with_srid(geom),
|
||||
'country_code': country,
|
||||
'indexed_status': indexed_status, 'indexed_date': indexed_date,
|
||||
'partition': pysql.Literal(0), 'geometry_sector': pysql.Literal(1)}
|
||||
return temp_db_cursor.insert_row('placex', **args)
|
||||
|
||||
return _add
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def osmline_table(temp_db_with_extensions, load_sql):
|
||||
load_sql('tables/interpolation.sql')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def osmline_row(osmline_table, temp_db_cursor):
|
||||
idseq = itertools.count(20001)
|
||||
|
||||
def _add(osm_id=None, geom='LINESTRING(12.0 11.0, 12.003 11.0)'):
|
||||
return temp_db_cursor.insert_row(
|
||||
'location_property_osmline',
|
||||
place_id=pysql.SQL("nextval('seq_place')"),
|
||||
osm_id=osm_id or next(idseq),
|
||||
geometry_sector=pysql.Literal(20),
|
||||
partition=pysql.Literal(0),
|
||||
indexed_status=1,
|
||||
linegeo=_with_srid(geom))
|
||||
|
||||
return _add
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def postcode_table(temp_db_with_extensions, load_sql):
|
||||
load_sql('tables/postcodes.sql')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def postcode_row(postcode_table, temp_db_cursor):
|
||||
def _add(country, postcode, x=34.5, y=-9.33):
|
||||
geom = _with_srid(f"POINT({x} {y})")
|
||||
return temp_db_cursor.insert_row(
|
||||
'location_postcodes',
|
||||
place_id=pysql.SQL("nextval('seq_place')"),
|
||||
indexed_status=pysql.Literal(1),
|
||||
country_code=country, postcode=postcode,
|
||||
centroid=geom,
|
||||
rank_search=pysql.Literal(16),
|
||||
geometry=('ST_Expand(%s::geometry, 0.005)', geom))
|
||||
|
||||
return _add
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sql_preprocessor_cfg(tmp_path, table_factory, temp_db_with_extensions, country_row):
|
||||
for part in range(3):
|
||||
country_row(partition=part)
|
||||
|
||||
cfg = Configuration(None)
|
||||
cfg.set_libdirs(sql=tmp_path)
|
||||
return cfg
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Specialised psycopg cursor with shortcut functions useful for testing.
|
||||
"""
|
||||
import psycopg
|
||||
from psycopg import sql as pysql
|
||||
|
||||
|
||||
class CursorForTesting(psycopg.Cursor):
|
||||
@@ -52,7 +53,49 @@ class CursorForTesting(psycopg.Cursor):
|
||||
def table_rows(self, table, where=None):
|
||||
""" Return the number of rows in the given table.
|
||||
"""
|
||||
if where is None:
|
||||
return self.scalar('SELECT count(*) FROM ' + table)
|
||||
sql = pysql.SQL('SELECT count(*) FROM') + pysql.Identifier(table)
|
||||
if where is not None:
|
||||
sql += pysql.SQL('WHERE') + pysql.SQL(where)
|
||||
|
||||
return self.scalar('SELECT count(*) FROM {} WHERE {}'.format(table, where))
|
||||
return self.scalar(sql)
|
||||
|
||||
def insert_row(self, table, **data):
|
||||
""" Insert a row into the given table.
|
||||
|
||||
'data' is a dictionary of column names and associated values.
|
||||
When the value is a pysql.Literal or pysql.SQL, then the expression
|
||||
will be inserted as is instead of loading the value. When the
|
||||
value is a tuple, then the first element will be added as an
|
||||
SQL expression for the value and the second element is treated
|
||||
as the actual value to insert. The SQL expression must contain
|
||||
a %s placeholder in that case.
|
||||
|
||||
If data contains a 'place_id' column, then the value of the
|
||||
place_id column after insert is returned. Otherwise the function
|
||||
returns nothing.
|
||||
"""
|
||||
columns = []
|
||||
placeholders = []
|
||||
values = []
|
||||
for k, v in data.items():
|
||||
columns.append(pysql.Identifier(k))
|
||||
if isinstance(v, tuple):
|
||||
placeholders.append(pysql.SQL(v[0]))
|
||||
values.append(v[1])
|
||||
elif isinstance(v, (pysql.Literal, pysql.SQL)):
|
||||
placeholders.append(v)
|
||||
else:
|
||||
placeholders.append(pysql.Placeholder())
|
||||
values.append(v)
|
||||
|
||||
sql = pysql.SQL("INSERT INTO {table} ({columns}) VALUES({values})")\
|
||||
.format(table=pysql.Identifier(table),
|
||||
columns=pysql.SQL(',').join(columns),
|
||||
values=pysql.SQL(',').join(placeholders))
|
||||
|
||||
if 'place_id' in data:
|
||||
sql += pysql.SQL('RETURNING place_id')
|
||||
|
||||
self.execute(sql, values)
|
||||
|
||||
return self.fetchone()[0] if 'place_id' in data else None
|
||||
|
||||
@@ -22,7 +22,8 @@ def loaded_country(def_config):
|
||||
def env_with_country_config(project_env):
|
||||
|
||||
def _mk_config(cfg):
|
||||
(project_env.project_dir / 'country_settings.yaml').write_text(dedent(cfg))
|
||||
(project_env.project_dir / 'country_settings.yaml').write_text(
|
||||
dedent(cfg), encoding='utf-8')
|
||||
|
||||
return project_env
|
||||
|
||||
@@ -52,11 +53,10 @@ def test_setup_country_tables(src_dir, temp_db_with_extensions, dsn, temp_db_cur
|
||||
|
||||
@pytest.mark.parametrize("languages", (None, ['fr', 'en']))
|
||||
def test_create_country_names(temp_db_with_extensions, temp_db_conn, temp_db_cursor,
|
||||
table_factory, tokenizer_mock, languages, loaded_country):
|
||||
|
||||
table_factory('country_name', 'country_code varchar(2), name hstore',
|
||||
content=(('us', '"name"=>"us1","name:af"=>"us2"'),
|
||||
('fr', '"name"=>"Fra", "name:en"=>"Fren"')))
|
||||
country_row, tokenizer_mock, languages, loaded_country):
|
||||
temp_db_cursor.execute('TRUNCATE country_name')
|
||||
country_row(country='us', names={"name": "us1", "name:af": "us2"})
|
||||
country_row(country='fr', names={"name": "Fra", "name:en": "Fren"})
|
||||
|
||||
assert temp_db_cursor.scalar("SELECT count(*) FROM country_name") == 2
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ def sql_factory(tmp_path):
|
||||
BEGIN
|
||||
{}
|
||||
END;
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;""".format(sql_body))
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;""".format(sql_body), encoding='utf-8')
|
||||
return 'test.sql'
|
||||
|
||||
return _mk_sql
|
||||
@@ -63,7 +63,7 @@ def test_load_file_with_params(sql_preprocessor, sql_factory, temp_db_conn, temp
|
||||
async def test_load_parallel_file(dsn, sql_preprocessor, tmp_path, temp_db_cursor):
|
||||
(tmp_path / 'test.sql').write_text("""
|
||||
CREATE TABLE foo (a TEXT);
|
||||
CREATE TABLE foo2(a TEXT);""" + "\n---\nCREATE TABLE bar (b INT);")
|
||||
CREATE TABLE foo2(a TEXT);""" + "\n---\nCREATE TABLE bar (b INT);", encoding='utf-8')
|
||||
|
||||
await sql_preprocessor.run_parallel_sql_file(dsn, 'test.sql', num_threads=4)
|
||||
|
||||
|
||||
@@ -15,7 +15,8 @@ from nominatim_db.errors import UsageError
|
||||
|
||||
def test_execute_file_success(dsn, temp_db_cursor, tmp_path):
|
||||
tmpfile = tmp_path / 'test.sql'
|
||||
tmpfile.write_text('CREATE TABLE test (id INT);\nINSERT INTO test VALUES(56);')
|
||||
tmpfile.write_text(
|
||||
'CREATE TABLE test (id INT);\nINSERT INTO test VALUES(56);', encoding='utf-8')
|
||||
|
||||
db_utils.execute_file(dsn, tmpfile)
|
||||
|
||||
@@ -29,7 +30,7 @@ def test_execute_file_bad_file(dsn, tmp_path):
|
||||
|
||||
def test_execute_file_bad_sql(dsn, tmp_path):
|
||||
tmpfile = tmp_path / 'test.sql'
|
||||
tmpfile.write_text('CREATE STABLE test (id INT)')
|
||||
tmpfile.write_text('CREATE STABLE test (id INT)', encoding='utf-8')
|
||||
|
||||
with pytest.raises(UsageError):
|
||||
db_utils.execute_file(dsn, tmpfile)
|
||||
@@ -37,14 +38,14 @@ def test_execute_file_bad_sql(dsn, tmp_path):
|
||||
|
||||
def test_execute_file_bad_sql_ignore_errors(dsn, tmp_path):
|
||||
tmpfile = tmp_path / 'test.sql'
|
||||
tmpfile.write_text('CREATE STABLE test (id INT)')
|
||||
tmpfile.write_text('CREATE STABLE test (id INT)', encoding='utf-8')
|
||||
|
||||
db_utils.execute_file(dsn, tmpfile, ignore_errors=True)
|
||||
|
||||
|
||||
def test_execute_file_with_pre_code(dsn, tmp_path, temp_db_cursor):
|
||||
tmpfile = tmp_path / 'test.sql'
|
||||
tmpfile.write_text('INSERT INTO test VALUES(4)')
|
||||
tmpfile.write_text('INSERT INTO test VALUES(4)', encoding='utf-8')
|
||||
|
||||
db_utils.execute_file(dsn, tmpfile, pre_code='CREATE TABLE test (id INT)')
|
||||
|
||||
@@ -53,7 +54,7 @@ def test_execute_file_with_pre_code(dsn, tmp_path, temp_db_cursor):
|
||||
|
||||
def test_execute_file_with_post_code(dsn, tmp_path, temp_db_cursor):
|
||||
tmpfile = tmp_path / 'test.sql'
|
||||
tmpfile.write_text('CREATE TABLE test (id INT)')
|
||||
tmpfile.write_text('CREATE TABLE test (id INT)', encoding='utf-8')
|
||||
|
||||
db_utils.execute_file(dsn, tmpfile, post_code='INSERT INTO test VALUES(23)')
|
||||
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Tests for running the indexing.
|
||||
"""
|
||||
import itertools
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio # noqa
|
||||
|
||||
@@ -15,129 +14,57 @@ from nominatim_db.indexer import indexer
|
||||
from nominatim_db.tokenizer import factory
|
||||
|
||||
|
||||
class IndexerTestDB:
|
||||
class TestIndexing:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup(self, temp_db_conn, project_env, tokenizer_mock,
|
||||
placex_table, postcode_table, osmline_table):
|
||||
self.conn = temp_db_conn
|
||||
temp_db_conn.execute("""
|
||||
CREATE OR REPLACE FUNCTION date_update() RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
|
||||
NEW.indexed_date = now();
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END; $$ LANGUAGE plpgsql;
|
||||
|
||||
def __init__(self, conn):
|
||||
self.placex_id = itertools.count(100000)
|
||||
self.osmline_id = itertools.count(500000)
|
||||
self.postcode_id = itertools.count(700000)
|
||||
DROP TYPE IF EXISTS prepare_update_info CASCADE;
|
||||
CREATE TYPE prepare_update_info AS (
|
||||
name HSTORE,
|
||||
address HSTORE,
|
||||
rank_address SMALLINT,
|
||||
country_code TEXT,
|
||||
class TEXT,
|
||||
type TEXT,
|
||||
linked_place_id BIGINT
|
||||
);
|
||||
CREATE OR REPLACE FUNCTION placex_indexing_prepare(p placex,
|
||||
OUT result prepare_update_info) AS $$
|
||||
BEGIN
|
||||
result.address := p.address;
|
||||
result.name := p.name;
|
||||
result.class := p.class;
|
||||
result.type := p.type;
|
||||
result.country_code := p.country_code;
|
||||
result.rank_address := p.rank_address;
|
||||
END; $$ LANGUAGE plpgsql STABLE;
|
||||
|
||||
self.conn = conn
|
||||
self.conn.autocimmit = True
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""CREATE TABLE placex (place_id BIGINT,
|
||||
name HSTORE,
|
||||
class TEXT,
|
||||
type TEXT,
|
||||
linked_place_id BIGINT,
|
||||
rank_address SMALLINT,
|
||||
rank_search SMALLINT,
|
||||
indexed_status SMALLINT,
|
||||
indexed_date TIMESTAMP,
|
||||
partition SMALLINT,
|
||||
admin_level SMALLINT,
|
||||
country_code TEXT,
|
||||
address HSTORE,
|
||||
token_info JSONB,
|
||||
geometry_sector INTEGER)""")
|
||||
cur.execute("""CREATE TABLE location_property_osmline (
|
||||
place_id BIGINT,
|
||||
osm_id BIGINT,
|
||||
address HSTORE,
|
||||
token_info JSONB,
|
||||
indexed_status SMALLINT,
|
||||
indexed_date TIMESTAMP,
|
||||
geometry_sector INTEGER)""")
|
||||
cur.execute("""CREATE TABLE location_postcodes (
|
||||
place_id BIGINT,
|
||||
indexed_status SMALLINT,
|
||||
indexed_date TIMESTAMP,
|
||||
country_code varchar(2),
|
||||
postcode TEXT)""")
|
||||
cur.execute("""CREATE OR REPLACE FUNCTION date_update() RETURNS TRIGGER
|
||||
AS $$
|
||||
BEGIN
|
||||
IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
|
||||
NEW.indexed_date = now();
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END; $$ LANGUAGE plpgsql;""")
|
||||
cur.execute("DROP TYPE IF EXISTS prepare_update_info CASCADE")
|
||||
cur.execute("""CREATE TYPE prepare_update_info AS (
|
||||
name HSTORE,
|
||||
address HSTORE,
|
||||
rank_address SMALLINT,
|
||||
country_code TEXT,
|
||||
class TEXT,
|
||||
type TEXT,
|
||||
linked_place_id BIGINT
|
||||
)""")
|
||||
cur.execute("""CREATE OR REPLACE FUNCTION placex_indexing_prepare(p placex,
|
||||
OUT result prepare_update_info)
|
||||
AS $$
|
||||
BEGIN
|
||||
result.address := p.address;
|
||||
result.name := p.name;
|
||||
result.class := p.class;
|
||||
result.type := p.type;
|
||||
result.country_code := p.country_code;
|
||||
result.rank_address := p.rank_address;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql STABLE;
|
||||
""")
|
||||
cur.execute("""CREATE OR REPLACE FUNCTION
|
||||
get_interpolation_address(in_address HSTORE, wayid BIGINT)
|
||||
RETURNS HSTORE AS $$
|
||||
BEGIN
|
||||
RETURN in_address;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql STABLE;
|
||||
""")
|
||||
CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT)
|
||||
RETURNS HSTORE AS $$ SELECT in_address $$ LANGUAGE sql STABLE;
|
||||
""")
|
||||
|
||||
for table in ('placex', 'location_property_osmline', 'location_postcodes'):
|
||||
cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
|
||||
FOR EACH ROW EXECUTE PROCEDURE date_update()
|
||||
""".format(table))
|
||||
for table in ('placex', 'location_property_osmline', 'location_postcodes'):
|
||||
temp_db_conn.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
|
||||
FOR EACH ROW EXECUTE PROCEDURE date_update()
|
||||
""".format(table))
|
||||
|
||||
self.tokenizer = factory.create_tokenizer(project_env)
|
||||
|
||||
def scalar(self, query):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(query)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
def add_place(self, cls='place', typ='locality',
|
||||
rank_search=30, rank_address=30, sector=20):
|
||||
next_id = next(self.placex_id)
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO placex
|
||||
(place_id, class, type, rank_search, rank_address,
|
||||
indexed_status, geometry_sector)
|
||||
VALUES (%s, %s, %s, %s, %s, 1, %s)""",
|
||||
(next_id, cls, typ, rank_search, rank_address, sector))
|
||||
return next_id
|
||||
|
||||
def add_admin(self, **kwargs):
|
||||
kwargs['cls'] = 'boundary'
|
||||
kwargs['typ'] = 'administrative'
|
||||
return self.add_place(**kwargs)
|
||||
|
||||
def add_osmline(self, sector=20):
|
||||
next_id = next(self.osmline_id)
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO location_property_osmline
|
||||
(place_id, osm_id, indexed_status, geometry_sector)
|
||||
VALUES (%s, %s, 1, %s)""",
|
||||
(next_id, next_id, sector))
|
||||
return next_id
|
||||
|
||||
def add_postcode(self, country, postcode):
|
||||
next_id = next(self.postcode_id)
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO location_postcodes
|
||||
(place_id, indexed_status, country_code, postcode)
|
||||
VALUES (%s, 1, %s, %s)""",
|
||||
(next_id, country, postcode))
|
||||
return next_id
|
||||
|
||||
def placex_unindexed(self):
|
||||
return self.scalar('SELECT count(*) from placex where indexed_status > 0')
|
||||
|
||||
@@ -145,148 +72,133 @@ class IndexerTestDB:
|
||||
return self.scalar("""SELECT count(*) from location_property_osmline
|
||||
WHERE indexed_status > 0""")
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_all_by_rank(self, dsn, threads, placex_row, osmline_row):
|
||||
for rank in range(31):
|
||||
placex_row(rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
osmline_row()
|
||||
|
||||
@pytest.fixture
|
||||
def test_db(temp_db_conn):
|
||||
yield IndexerTestDB(temp_db_conn)
|
||||
assert self.placex_unindexed() == 31
|
||||
assert self.osmline_unindexed() == 1
|
||||
|
||||
idx = indexer.Indexer(dsn, self.tokenizer, threads)
|
||||
await idx.index_by_rank(0, 30)
|
||||
|
||||
@pytest.fixture
|
||||
def test_tokenizer(tokenizer_mock, project_env):
|
||||
return factory.create_tokenizer(project_env)
|
||||
assert self.placex_unindexed() == 0
|
||||
assert self.osmline_unindexed() == 0
|
||||
|
||||
assert self.scalar("""SELECT count(*) from placex
|
||||
WHERE indexed_status = 0 and indexed_date is null""") == 0
|
||||
# ranks come in order of rank address
|
||||
assert self.scalar("""
|
||||
SELECT count(*) FROM placex p WHERE rank_address > 0
|
||||
AND indexed_date >= (SELECT min(indexed_date) FROM placex o
|
||||
WHERE p.rank_address < o.rank_address)""") == 0
|
||||
# placex address ranked objects come before interpolations
|
||||
assert self.scalar(
|
||||
"""SELECT count(*) FROM placex WHERE rank_address > 0
|
||||
AND indexed_date >
|
||||
(SELECT min(indexed_date) FROM location_property_osmline)""") == 0
|
||||
# rank 0 comes after all other placex objects
|
||||
assert self.scalar(
|
||||
"""SELECT count(*) FROM placex WHERE rank_address > 0
|
||||
AND indexed_date >
|
||||
(SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_all_by_rank(test_db, threads, test_tokenizer):
|
||||
for rank in range(31):
|
||||
test_db.add_place(rank_address=rank, rank_search=rank)
|
||||
test_db.add_osmline()
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_partial_without_30(self, dsn, threads, placex_row, osmline_row):
|
||||
for rank in range(31):
|
||||
placex_row(rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
osmline_row()
|
||||
|
||||
assert test_db.placex_unindexed() == 31
|
||||
assert test_db.osmline_unindexed() == 1
|
||||
assert self.placex_unindexed() == 31
|
||||
assert self.osmline_unindexed() == 1
|
||||
|
||||
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
|
||||
await idx.index_by_rank(0, 30)
|
||||
idx = indexer.Indexer(dsn, self.tokenizer, threads)
|
||||
await idx.index_by_rank(4, 15)
|
||||
|
||||
assert test_db.placex_unindexed() == 0
|
||||
assert test_db.osmline_unindexed() == 0
|
||||
assert self.placex_unindexed() == 19
|
||||
assert self.osmline_unindexed() == 1
|
||||
|
||||
assert test_db.scalar("""SELECT count(*) from placex
|
||||
WHERE indexed_status = 0 and indexed_date is null""") == 0
|
||||
# ranks come in order of rank address
|
||||
assert test_db.scalar("""
|
||||
SELECT count(*) FROM placex p WHERE rank_address > 0
|
||||
AND indexed_date >= (SELECT min(indexed_date) FROM placex o
|
||||
WHERE p.rank_address < o.rank_address)""") == 0
|
||||
# placex address ranked objects come before interpolations
|
||||
assert test_db.scalar(
|
||||
"""SELECT count(*) FROM placex WHERE rank_address > 0
|
||||
AND indexed_date >
|
||||
(SELECT min(indexed_date) FROM location_property_osmline)""") == 0
|
||||
# rank 0 comes after all other placex objects
|
||||
assert test_db.scalar(
|
||||
"""SELECT count(*) FROM placex WHERE rank_address > 0
|
||||
AND indexed_date >
|
||||
(SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
|
||||
assert self.scalar("""
|
||||
SELECT count(*) FROM placex
|
||||
WHERE indexed_status = 0 AND not rank_address between 4 and 15""") == 0
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_partial_with_30(self, dsn, threads, placex_row, osmline_row):
|
||||
for rank in range(31):
|
||||
placex_row(rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
osmline_row()
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_partial_without_30(test_db, threads, test_tokenizer):
|
||||
for rank in range(31):
|
||||
test_db.add_place(rank_address=rank, rank_search=rank)
|
||||
test_db.add_osmline()
|
||||
assert self.placex_unindexed() == 31
|
||||
assert self.osmline_unindexed() == 1
|
||||
|
||||
assert test_db.placex_unindexed() == 31
|
||||
assert test_db.osmline_unindexed() == 1
|
||||
idx = indexer.Indexer(dsn, self.tokenizer, threads)
|
||||
await idx.index_by_rank(28, 30)
|
||||
|
||||
idx = indexer.Indexer('dbname=test_nominatim_python_unittest',
|
||||
test_tokenizer, threads)
|
||||
await idx.index_by_rank(4, 15)
|
||||
assert self.placex_unindexed() == 28
|
||||
assert self.osmline_unindexed() == 0
|
||||
|
||||
assert test_db.placex_unindexed() == 19
|
||||
assert test_db.osmline_unindexed() == 1
|
||||
assert self.scalar("""
|
||||
SELECT count(*) FROM placex
|
||||
WHERE indexed_status = 0 AND rank_address between 0 and 27""") == 0
|
||||
|
||||
assert test_db.scalar("""
|
||||
SELECT count(*) FROM placex
|
||||
WHERE indexed_status = 0 AND not rank_address between 4 and 15""") == 0
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_boundaries(self, dsn, threads, placex_row, osmline_row):
|
||||
for rank in range(4, 10):
|
||||
placex_row(cls='boundary', typ='administrative',
|
||||
rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
for rank in range(31):
|
||||
placex_row(rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
osmline_row()
|
||||
|
||||
assert self.placex_unindexed() == 37
|
||||
assert self.osmline_unindexed() == 1
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_partial_with_30(test_db, threads, test_tokenizer):
|
||||
for rank in range(31):
|
||||
test_db.add_place(rank_address=rank, rank_search=rank)
|
||||
test_db.add_osmline()
|
||||
idx = indexer.Indexer(dsn, self.tokenizer, threads)
|
||||
await idx.index_boundaries()
|
||||
|
||||
assert test_db.placex_unindexed() == 31
|
||||
assert test_db.osmline_unindexed() == 1
|
||||
assert self.placex_unindexed() == 31
|
||||
assert self.osmline_unindexed() == 1
|
||||
|
||||
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
|
||||
await idx.index_by_rank(28, 30)
|
||||
assert self.scalar("""
|
||||
SELECT count(*) FROM placex
|
||||
WHERE indexed_status = 0 AND class != 'boundary'""") == 0
|
||||
|
||||
assert test_db.placex_unindexed() == 27
|
||||
assert test_db.osmline_unindexed() == 0
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_postcodes(self, dsn, threads, postcode_row):
|
||||
for postcode in range(1000):
|
||||
postcode_row(country='de', postcode=postcode)
|
||||
for postcode in range(32000, 33000):
|
||||
postcode_row(country='us', postcode=postcode)
|
||||
|
||||
assert test_db.scalar("""
|
||||
SELECT count(*) FROM placex
|
||||
WHERE indexed_status = 0 AND rank_address between 1 and 27""") == 0
|
||||
idx = indexer.Indexer(dsn, self.tokenizer, threads)
|
||||
await idx.index_postcodes()
|
||||
|
||||
assert self.scalar("""SELECT count(*) FROM location_postcodes
|
||||
WHERE indexed_status != 0""") == 0
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_boundaries(test_db, threads, test_tokenizer):
|
||||
for rank in range(4, 10):
|
||||
test_db.add_admin(rank_address=rank, rank_search=rank)
|
||||
for rank in range(31):
|
||||
test_db.add_place(rank_address=rank, rank_search=rank)
|
||||
test_db.add_osmline()
|
||||
@pytest.mark.parametrize("analyse", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_full(self, dsn, analyse, placex_row, osmline_row, postcode_row):
|
||||
for rank in range(4, 10):
|
||||
placex_row(cls='boundary', typ='administrative',
|
||||
rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
for rank in range(31):
|
||||
placex_row(rank_address=rank, rank_search=rank, indexed_status=1)
|
||||
osmline_row()
|
||||
for postcode in range(1000):
|
||||
postcode_row(country='de', postcode=postcode)
|
||||
|
||||
assert test_db.placex_unindexed() == 37
|
||||
assert test_db.osmline_unindexed() == 1
|
||||
idx = indexer.Indexer(dsn, self.tokenizer, 4)
|
||||
await idx.index_full(analyse=analyse)
|
||||
|
||||
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
|
||||
await idx.index_boundaries(0, 30)
|
||||
|
||||
assert test_db.placex_unindexed() == 31
|
||||
assert test_db.osmline_unindexed() == 1
|
||||
|
||||
assert test_db.scalar("""
|
||||
SELECT count(*) FROM placex
|
||||
WHERE indexed_status = 0 AND class != 'boundary'""") == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("threads", [1, 15])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_postcodes(test_db, threads, test_tokenizer):
|
||||
for postcode in range(1000):
|
||||
test_db.add_postcode('de', postcode)
|
||||
for postcode in range(32000, 33000):
|
||||
test_db.add_postcode('us', postcode)
|
||||
|
||||
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
|
||||
await idx.index_postcodes()
|
||||
|
||||
assert test_db.scalar("""SELECT count(*) FROM location_postcodes
|
||||
WHERE indexed_status != 0""") == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("analyse", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_full(test_db, analyse, test_tokenizer):
|
||||
for rank in range(4, 10):
|
||||
test_db.add_admin(rank_address=rank, rank_search=rank)
|
||||
for rank in range(31):
|
||||
test_db.add_place(rank_address=rank, rank_search=rank)
|
||||
test_db.add_osmline()
|
||||
for postcode in range(1000):
|
||||
test_db.add_postcode('de', postcode)
|
||||
|
||||
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4)
|
||||
await idx.index_full(analyse=analyse)
|
||||
|
||||
assert test_db.placex_unindexed() == 0
|
||||
assert test_db.osmline_unindexed() == 0
|
||||
assert test_db.scalar("""SELECT count(*) FROM location_postcodes
|
||||
WHERE indexed_status != 0""") == 0
|
||||
assert self.placex_unindexed() == 0
|
||||
assert self.osmline_unindexed() == 0
|
||||
assert self.scalar("""SELECT count(*) FROM location_postcodes
|
||||
WHERE indexed_status != 0""") == 0
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Custom mocks for testing.
|
||||
"""
|
||||
import itertools
|
||||
|
||||
from nominatim_db.db import properties
|
||||
|
||||
|
||||
class MockPlacexTable:
|
||||
""" A placex table for testing.
|
||||
"""
|
||||
def __init__(self, conn):
|
||||
self.idseq = itertools.count(10000)
|
||||
self.conn = conn
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""CREATE TABLE placex (
|
||||
place_id BIGINT,
|
||||
parent_place_id BIGINT,
|
||||
linked_place_id BIGINT,
|
||||
importance FLOAT,
|
||||
indexed_date TIMESTAMP,
|
||||
geometry_sector INTEGER,
|
||||
rank_address SMALLINT,
|
||||
rank_search SMALLINT,
|
||||
partition SMALLINT,
|
||||
indexed_status SMALLINT,
|
||||
osm_id int8,
|
||||
osm_type char(1),
|
||||
class text,
|
||||
type text,
|
||||
name hstore,
|
||||
admin_level smallint,
|
||||
address hstore,
|
||||
extratags hstore,
|
||||
token_info jsonb,
|
||||
geometry Geometry(Geometry,4326),
|
||||
wikipedia TEXT,
|
||||
country_code varchar(2),
|
||||
housenumber TEXT,
|
||||
postcode TEXT,
|
||||
centroid GEOMETRY(Geometry, 4326))""")
|
||||
cur.execute("CREATE SEQUENCE IF NOT EXISTS seq_place")
|
||||
conn.commit()
|
||||
|
||||
def add(self, osm_type='N', osm_id=None, cls='amenity', typ='cafe', names=None,
|
||||
admin_level=None, address=None, extratags=None, geom='POINT(10 4)',
|
||||
country=None, housenumber=None, rank_search=30, centroid=None):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO placex (place_id, osm_type, osm_id, class,
|
||||
type, name, admin_level, address,
|
||||
housenumber, rank_search,
|
||||
extratags, centroid, geometry, country_code)
|
||||
VALUES(nextval('seq_place'), %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING place_id""",
|
||||
(osm_type, osm_id or next(self.idseq), cls, typ, names,
|
||||
admin_level, address, housenumber, rank_search,
|
||||
extratags, centroid, 'SRID=4326;' + geom,
|
||||
country))
|
||||
place_id = cur.fetchone()[0]
|
||||
self.conn.commit()
|
||||
return place_id
|
||||
|
||||
|
||||
class MockPropertyTable:
|
||||
""" A property table for testing.
|
||||
"""
|
||||
def __init__(self, conn):
|
||||
self.conn = conn
|
||||
|
||||
def set(self, name, value):
|
||||
""" Set a property in the table to the given value.
|
||||
"""
|
||||
properties.set_property(self.conn, name, value)
|
||||
|
||||
def get(self, name):
|
||||
""" Set a property in the table to the given value.
|
||||
"""
|
||||
return properties.get_property(self.conn, name)
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# This file is part of Nominatim. (https://nominatim.org)
|
||||
#
|
||||
# Copyright (C) 2025 by the Nominatim developer community.
|
||||
# Copyright (C) 2026 by the Nominatim developer community.
|
||||
# For a full list of authors see the git log.
|
||||
"""
|
||||
Tests for the sanitizer that normalizes housenumbers.
|
||||
@@ -67,3 +67,25 @@ def test_convert_to_name_unconverted(def_config, number):
|
||||
|
||||
assert 'housenumber' not in set(p.kind for p in names)
|
||||
assert ('housenumber', number) in set((p.kind, p.name) for p in address)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('hnr,itype,out', [
|
||||
('1-5', 'all', (1, 2, 3, 4, 5)),
|
||||
('1-5', 'odd', (1, 3, 5)),
|
||||
('1-5', 'even', (2, 4)),
|
||||
('6-9', '1', (6, 7, 8, 9)),
|
||||
('6-9', '2', (6, 8)),
|
||||
('6-9', '3', (6, 9)),
|
||||
('6-9', '5', (6,)),
|
||||
('6-9', 'odd', (7, 9)),
|
||||
('6-9', 'even', (6, 8)),
|
||||
('6-22', 'even', (6, 8, 10, 12, 14, 16, 18, 20, 22))
|
||||
])
|
||||
def test_convert_interpolations(sanitize, hnr, itype, out):
|
||||
assert set(sanitize(housenumber=hnr, interpolation=itype)) \
|
||||
== {('housenumber', str(i)) for i in out}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('hnr', ('23', '23-', '3z-f', '1-10', '5-1', '1-4-5'))
|
||||
def test_ignore_interpolation_with_bad_housenumber(sanitize, hnr):
|
||||
assert sanitize(housenumber=hnr, interpolation='all') == [('housenumber', hnr)]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user