Clean and update tests for postcodes

nominatim/tools/postcodes.py

@@ -9,7 +9,7 @@ from math import isfinite
 
 from psycopg2.extras import execute_values
 
-from nominatim.db.connection import _Connection, connect
+from nominatim.db.connection import connect
 
 LOG = logging.getLogger()
 
@@ -169,7 +169,7 @@ def update_postcodes(dsn, project_dir, tokenizer):
                     SELECT 
                         COALESCE(plx.country_code, get_country_code(ST_Centroid(pl.geometry))) as cc,
                         token_normalized_postcode(pl.address->'postcode') as pc,
-                        ST_Centroid(ST_Collect(ST_Centroid(pl.geometry))) as centroid
+                        COALESCE(ST_Centroid(ST_Collect(plx.centroid)), ST_Centroid(ST_Collect(ST_Centroid(pl.geometry)))) as centroid
                     FROM place AS pl LEFT OUTER JOIN placex AS plx ON pl.osm_id = plx.osm_id AND pl.osm_type = plx.osm_type
                     WHERE pl.address ? 'postcode' AND pl.geometry IS NOT null
                     GROUP BY cc, pc
@@ -203,5 +203,5 @@ def can_compute(dsn):
         Check that the place table exists so that
         postcodes can be computed.
     """
-    with _Connection(dsn) as conn:
+    with connect(dsn) as conn:
         return conn.table_exists('place')

test/python/test_tools_postcodes.py

@@ -29,7 +29,7 @@ class MockPostcodeTable:
 
                            CREATE OR REPLACE FUNCTION get_country_code(place geometry)
                            RETURNS TEXT AS $$ BEGIN 
-                           RETURN (SELECT country_code FROM placex WHERE geometry = place LIMIT 1);
+                           RETURN null;
                            END; $$ LANGUAGE plpgsql;
                         """)
         conn.commit()
@@ -70,10 +70,9 @@ def test_postcodes_empty(dsn, postcode_table, place_table,
     assert not postcode_table.row_set
 
 
-def test_postcodes_add_new(dsn, postcode_table, placex_table, place_row,
+def test_postcodes_add_new(dsn, postcode_table, tmp_path,
-                           tmp_path, tokenizer):
+                           insert_implicit_postcode, tokenizer):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='9486'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='9486'))
     postcode_table.add('yy', '9486', 99, 34)
 
     postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -81,10 +80,9 @@ def test_postcodes_add_new(dsn, postcode_table, placex_table, place_row,
     assert postcode_table.row_set == {('xx', '9486', 10, 12), }
 
 
-def test_postcodes_replace_coordinates(dsn, placex_table, postcode_table,
+def test_postcodes_replace_coordinates(dsn, postcode_table, tmp_path,
-                                       place_row, tmp_path, tokenizer):
+                                       insert_implicit_postcode, tokenizer):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
     postcode_table.add('xx', 'AB 4511', 99, 34)
 
     postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -92,10 +90,9 @@ def test_postcodes_replace_coordinates(dsn, placex_table, postcode_table,
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
 
 
-def test_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table,
+def test_postcodes_replace_coordinates_close(dsn, postcode_table, tmp_path,
-                                             place_row, tmp_path, tokenizer):
+                                             insert_implicit_postcode, tokenizer):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
     postcode_table.add('xx', 'AB 4511', 10, 11.99999)
 
     postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -103,10 +100,9 @@ def test_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table,
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 11.99999)}
 
 
-def test_postcodes_remove(dsn, placex_table, postcode_table, 
+def test_postcodes_remove(dsn, postcode_table, tmp_path,
-                          place_row, tmp_path, tokenizer):
+                          insert_implicit_postcode, tokenizer):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
     postcode_table.add('xx', 'badname', 10, 12)
 
     postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@@ -114,12 +110,11 @@ def test_postcodes_remove(dsn, placex_table, postcode_table,
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
 
 
-def test_postcodes_ignore_empty_country(dsn, placex_table, postcode_table, 
+def test_postcodes_ignore_empty_country(dsn, postcode_table, tmp_path,
-                                        place_row, tmp_path, tokenizer):
+                                        insert_implicit_postcode, tokenizer):
-    placex_table.add(country=None, geom='POINT(10 12)')
+    insert_implicit_postcode(1, None, 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
     postcodes.update_postcodes(dsn, tmp_path, tokenizer)
-
+    print(postcode_table.row_set)
     assert not postcode_table.row_set
 
 
@@ -131,33 +126,25 @@ def test_postcodes_remove_all(dsn, postcode_table, place_table,
     assert not postcode_table.row_set
 
 
-def test_postcodes_multi_country(dsn, placex_table, postcode_table, 
+def test_postcodes_multi_country(dsn, postcode_table, tmp_path,
-                                 place_row, tmp_path, tokenizer):
+                                 insert_implicit_postcode, tokenizer):
-    placex_table.add(country='de', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'de', 'POINT(10 12)', dict(postcode='54451'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='54451'))
+    insert_implicit_postcode(2, 'cc', 'POINT(100 56)', dict(postcode='DD23 T'))
-
+    insert_implicit_postcode(3, 'de', 'POINT(10.3 11.0)', dict(postcode='54452'))
-    placex_table.add(country='cc', geom='POINT(100 56)')
+    insert_implicit_postcode(4, 'cc', 'POINT(10.3 11.0)', dict(postcode='54452'))
-    place_row(geom='SRID=4326;POINT(100 56)', address=dict(postcode='DD23 T'))
-
-    placex_table.add(country='de', geom='POINT(10.3 11.0)')
-    place_row(geom='SRID=4326;POINT(10.3 11.0)', address=dict(postcode='54452'))
-
-    placex_table.add(country='cc', geom='POINT(10.3 10.0)')
-    place_row(geom='SRID=4326;POINT(10.3 10.0)', address=dict(postcode='54452'))
 
     postcodes.update_postcodes(dsn, tmp_path, tokenizer)
 
     assert postcode_table.row_set == {('de', '54451', 10, 12),
                                       ('de', '54452', 10.3, 11.0),
-                                      ('cc', '54452', 10.3, 10.0),
+                                      ('cc', '54452', 10.3, 11.0),
                                       ('cc', 'DD23 T', 100, 56)}
 
 
 @pytest.mark.parametrize("gzipped", [True, False])
-def test_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
+def test_postcodes_extern(dsn, postcode_table, tmp_path,
-                          place_row, tokenizer, gzipped):
+                          insert_implicit_postcode, tokenizer, gzipped):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
 
     extfile = tmp_path / 'xx_postcodes.csv'
     extfile.write_text("postcode,lat,lon\nAB 4511,-4,-1\nCD 4511,-5, -10")
@@ -172,10 +159,9 @@ def test_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
                                       ('xx', 'CD 4511', -10, -5)}
 
 
-def test_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
+def test_postcodes_extern_bad_column(dsn, postcode_table, tmp_path, 
-                                     place_row, tmp_path, tokenizer):
+                                     insert_implicit_postcode, tokenizer):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
 
     extfile = tmp_path / 'xx_postcodes.csv'
     extfile.write_text("postode,lat,lon\nAB 4511,-4,-1\nCD 4511,-5, -10")
@@ -185,10 +171,9 @@ def test_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
 
 
-def test_postcodes_extern_bad_number(dsn, placex_table, postcode_table,
+def test_postcodes_extern_bad_number(dsn, insert_implicit_postcode,
-                                     place_row, tmp_path, tokenizer):
+                                     postcode_table, tmp_path, tokenizer):
-    placex_table.add(country='xx', geom='POINT(10 12)')
+    insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
-    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
 
     extfile = tmp_path / 'xx_postcodes.csv'
     extfile.write_text("postcode,lat,lon\nXX 4511,-4,NaN\nCD 4511,-5, -10\n34,200,0")
@@ -198,7 +183,32 @@ def test_postcodes_extern_bad_number(dsn, placex_table, postcode_table,
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12),
                                       ('xx', 'CD 4511', -10, -5)}
 
-def test_can_compute(dsn, temp_db_cursor):
+def test_can_compute(dsn, table_factory):
     assert not postcodes.can_compute(dsn)
-    temp_db_cursor.execute('CREATE TABLE place()')
+    table_factory('place')
     assert postcodes.can_compute(dsn)
+
+def test_no_placex_entry(dsn, tmp_path, temp_db_cursor, place_row, postcode_table, tokenizer):
+    #Rewrite the get_country_code function to verify its execution.
+    temp_db_cursor.execute("""
+        CREATE OR REPLACE FUNCTION get_country_code(place geometry)
+        RETURNS TEXT AS $$ BEGIN 
+        RETURN 'fr';
+        END; $$ LANGUAGE plpgsql;
+    """)
+    place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('fr', 'AB 4511', 10, 12)}
+
+@pytest.fixture
+def insert_implicit_postcode(placex_table, place_row):
+    """
+        Inserts data into the placex and place table
+        which can then be used to compute one postcode.
+    """
+    def _insert_implicit_postcode(osm_id, country, geometry, address):
+        placex_table.add(osm_id=osm_id, country=country, geom=geometry)
+        place_row(osm_id=osm_id, geom='SRID=4326;'+geometry, address=address)
+
+    return _insert_implicit_postcode