hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-14 18:37:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clean and update tests for postcodes

Browse Source
This commit is contained in:
AntoJvlt
2021-06-08 22:39:04 +02:00
committed by root
parent 47fb7cd3a8
commit 1c175e3a67
2 changed files with 63 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
nominatim/tools/postcodes.py
View File

@@ -9,7 +9,7 @@ from math import isfinite
from psycopg2.extras import execute_values
from nominatim.db.connection import _Connection, connect
from nominatim.db.connection import connect
LOG = logging.getLogger()
@@ -169,7 +169,7 @@ def update_postcodes(dsn, project_dir, tokenizer):
SELECT
COALESCE(plx.country_code, get_country_code(ST_Centroid(pl.geometry))) as cc,
token_normalized_postcode(pl.address->'postcode') as pc,
ST_Centroid(ST_Collect(ST_Centroid(pl.geometry))) as centroid
COALESCE(ST_Centroid(ST_Collect(plx.centroid)), ST_Centroid(ST_Collect(ST_Centroid(pl.geometry)))) as centroid
FROM place AS pl LEFT OUTER JOIN placex AS plx ON pl.osm_id = plx.osm_id AND pl.osm_type = plx.osm_type
WHERE pl.address ? 'postcode' AND pl.geometry IS NOT null
GROUP BY cc, pc
@@ -203,5 +203,5 @@ def can_compute(dsn):
Check that the place table exists so that
postcodes can be computed.
"""
with _Connection(dsn) as conn:
with connect(dsn) as conn:
return conn.table_exists('place')

110
test/python/test_tools_postcodes.py
View File

@@ -29,7 +29,7 @@ class MockPostcodeTable:
CREATE OR REPLACE FUNCTION get_country_code(place geometry)
RETURNS TEXT AS $$ BEGIN
RETURN (SELECT country_code FROM placex WHERE geometry = place LIMIT 1);
RETURN null;
END; $$ LANGUAGE plpgsql;
""")
conn.commit()
@@ -70,10 +70,9 @@ def test_postcodes_empty(dsn, postcode_table, place_table,
assert not postcode_table.row_set
def test_postcodes_add_new(dsn, postcode_table, placex_table, place_row,
tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='9486'))
def test_postcodes_add_new(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='9486'))
postcode_table.add('yy', '9486', 99, 34)
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -81,10 +80,9 @@ def test_postcodes_add_new(dsn, postcode_table, placex_table, place_row,
assert postcode_table.row_set == {('xx', '9486', 10, 12), }
def test_postcodes_replace_coordinates(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_replace_coordinates(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
postcode_table.add('xx', 'AB 4511', 99, 34)
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -92,10 +90,9 @@ def test_postcodes_replace_coordinates(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
def test_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_replace_coordinates_close(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
postcode_table.add('xx', 'AB 4511', 10, 11.99999)
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -103,10 +100,9 @@ def test_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 11.99999)}
def test_postcodes_remove(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_remove(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
postcode_table.add('xx', 'badname', 10, 12)
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -114,12 +110,11 @@ def test_postcodes_remove(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
def test_postcodes_ignore_empty_country(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country=None, geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_ignore_empty_country(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, None, 'POINT(10 12)', dict(postcode='AB 4511'))
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
print(postcode_table.row_set)
assert not postcode_table.row_set
@@ -131,33 +126,25 @@ def test_postcodes_remove_all(dsn, postcode_table, place_table,
assert not postcode_table.row_set
def test_postcodes_multi_country(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country='de', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='54451'))
placex_table.add(country='cc', geom='POINT(100 56)')
place_row(geom='SRID=4326;POINT(100 56)', address=dict(postcode='DD23 T'))
placex_table.add(country='de', geom='POINT(10.3 11.0)')
place_row(geom='SRID=4326;POINT(10.3 11.0)', address=dict(postcode='54452'))
placex_table.add(country='cc', geom='POINT(10.3 10.0)')
place_row(geom='SRID=4326;POINT(10.3 10.0)', address=dict(postcode='54452'))
def test_postcodes_multi_country(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, 'de', 'POINT(10 12)', dict(postcode='54451'))
insert_implicit_postcode(2, 'cc', 'POINT(100 56)', dict(postcode='DD23 T'))
insert_implicit_postcode(3, 'de', 'POINT(10.3 11.0)', dict(postcode='54452'))
insert_implicit_postcode(4, 'cc', 'POINT(10.3 11.0)', dict(postcode='54452'))
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
assert postcode_table.row_set == {('de', '54451', 10, 12),
('de', '54452', 10.3, 11.0),
('cc', '54452', 10.3, 10.0),
('cc', '54452', 10.3, 11.0),
('cc', 'DD23 T', 100, 56)}
@pytest.mark.parametrize("gzipped", [True, False])
def test_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
place_row, tokenizer, gzipped):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_extern(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer, gzipped):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
extfile = tmp_path / 'xx_postcodes.csv'
extfile.write_text("postcode,lat,lon\nAB 4511,-4,-1\nCD 4511,-5, -10")
@@ -172,10 +159,9 @@ def test_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
('xx', 'CD 4511', -10, -5)}
def test_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_extern_bad_column(dsn, postcode_table, tmp_path,
insert_implicit_postcode, tokenizer):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
extfile = tmp_path / 'xx_postcodes.csv'
extfile.write_text("postode,lat,lon\nAB 4511,-4,-1\nCD 4511,-5, -10")
@@ -185,10 +171,9 @@ def test_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
def test_postcodes_extern_bad_number(dsn, placex_table, postcode_table,
place_row, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)')
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
def test_postcodes_extern_bad_number(dsn, insert_implicit_postcode,
postcode_table, tmp_path, tokenizer):
insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
extfile = tmp_path / 'xx_postcodes.csv'
extfile.write_text("postcode,lat,lon\nXX 4511,-4,NaN\nCD 4511,-5, -10\n34,200,0")
@@ -198,7 +183,32 @@ def test_postcodes_extern_bad_number(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12),
('xx', 'CD 4511', -10, -5)}
def test_can_compute(dsn, temp_db_cursor):
def test_can_compute(dsn, table_factory):
assert not postcodes.can_compute(dsn)
temp_db_cursor.execute('CREATE TABLE place()')
table_factory('place')
assert postcodes.can_compute(dsn)
def test_no_placex_entry(dsn, tmp_path, temp_db_cursor, place_row, postcode_table, tokenizer):
#Rewrite the get_country_code function to verify its execution.
temp_db_cursor.execute("""
CREATE OR REPLACE FUNCTION get_country_code(place geometry)
RETURNS TEXT AS $$ BEGIN
RETURN 'fr';
END; $$ LANGUAGE plpgsql;
""")
place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
postcodes.update_postcodes(dsn, tmp_path, tokenizer)
assert postcode_table.row_set == {('fr', 'AB 4511', 10, 12)}
@pytest.fixture
def insert_implicit_postcode(placex_table, place_row):
"""
Inserts data into the placex and place table
which can then be used to compute one postcode.
"""
def _insert_implicit_postcode(osm_id, country, geometry, address):
placex_table.add(osm_id=osm_id, country=country, geom=geometry)
place_row(osm_id=osm_id, geom='SRID=4326;'+geometry, address=address)
return _insert_implicit_postcode