Resolve conflicts

test/bdd/steps/nominatim_environment.py

@@ -9,6 +9,7 @@ sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
 
 from nominatim import cli
 from nominatim.config import Configuration
+from nominatim.db.connection import _Connection
 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from steps.utils import run_script
@@ -54,7 +55,7 @@ class NominatimEnvironment:
             dbargs['user'] = self.db_user
         if self.db_pass:
             dbargs['password'] = self.db_pass
-        conn = psycopg2.connect(**dbargs)
+        conn = psycopg2.connect(connection_factory=_Connection, **dbargs)
         return conn
 
     def next_code_coverage_file(self):
@@ -110,8 +111,13 @@ class NominatimEnvironment:
             self.website_dir.cleanup()
 
         self.website_dir = tempfile.TemporaryDirectory()
+
+        try:
+            conn = self.connect_database(dbname)
+        except:
+            conn = False
         refresh.setup_website(Path(self.website_dir.name) / 'website',
-                              self.get_test_config())
+                              self.get_test_config(), conn)
 
 
     def get_test_config(self):
@@ -228,13 +234,13 @@ class NominatimEnvironment:
         """ Setup a test against a fresh, empty test database.
         """
         self.setup_template_db()
-        self.write_nominatim_config(self.test_db)
         conn = self.connect_database(self.template_db)
         conn.set_isolation_level(0)
         cur = conn.cursor()
         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
         conn.close()
+        self.write_nominatim_config(self.test_db)
         context.db = self.connect_database(self.test_db)
         context.db.autocommit = True
         psycopg2.extras.register_hstore(context.db, globally=False)

test/bdd/steps/steps_db_ops.py

@@ -251,7 +251,7 @@ def check_location_postcode(context):
     with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
         cur.execute("SELECT *, ST_AsText(geometry) as geomtxt FROM location_postcode")
         assert cur.rowcount == len(list(context.table)), \
-            "Postcode table has {} rows, expected {}.".foramt(cur.rowcount, len(list(context.table)))
+            "Postcode table has {} rows, expected {}.".format(cur.rowcount, len(list(context.table)))
 
         results = {}
         for row in cur:

test/python/conftest.py

@@ -19,6 +19,7 @@ from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.db import properties
 
 import dummy_tokenizer
+import mocks
 
 class _TestingCursor(psycopg2.extras.DictCursor):
     """ Extension to the DictCursor class that provides execution
@@ -211,33 +212,7 @@ def place_row(place_table, temp_db_cursor):
 def placex_table(temp_db_with_extensions, temp_db_conn):
     """ Create an empty version of the place table.
     """
-    with temp_db_conn.cursor() as cur:
-        cur.execute("""CREATE TABLE placex (
-                           place_id BIGINT,
-                           parent_place_id BIGINT,
-                           linked_place_id BIGINT,
-                           importance FLOAT,
-                           indexed_date TIMESTAMP,
-                           geometry_sector INTEGER,
-                           rank_address SMALLINT,
-                           rank_search SMALLINT,
-                           partition SMALLINT,
-                           indexed_status SMALLINT,
-                           osm_id int8,
-                           osm_type char(1),
-                           class text,
-                           type text,
-                           name hstore,
-                           admin_level smallint,
-                           address hstore,
-                           extratags hstore,
-                           geometry Geometry(Geometry,4326),
-                           wikipedia TEXT,
-                           country_code varchar(2),
-                           housenumber TEXT,
-                           postcode TEXT,
-                           centroid GEOMETRY(Geometry, 4326))""")
-    temp_db_conn.commit()
+    return mocks.MockPlacexTable(temp_db_conn)
 
 
 @pytest.fixture
@@ -262,18 +237,8 @@ def osmline_table(temp_db_with_extensions, temp_db_conn):
 
 
 @pytest.fixture
-def word_table(temp_db, temp_db_conn):
-    with temp_db_conn.cursor() as cur:
-        cur.execute("""CREATE TABLE word (
-                           word_id INTEGER,
-                           word_token text,
-                           word text,
-                           class text,
-                           type text,
-                           country_code varchar(2),
-                           search_name_count INTEGER,
-                           operator TEXT)""")
-    temp_db_conn.commit()
+def word_table(temp_db_conn):
+    return mocks.MockWordTable(temp_db_conn)
 
 
 @pytest.fixture

test/python/dummy_tokenizer.py

@@ -51,7 +51,10 @@ class DummyNameAnalyzer:
     def close(self):
         pass
 
-    def add_postcodes_from_db(self):
+    def normalize_postcode(self, postcode):
+        return postcode
+
+    def update_postcodes_from_db(self):
         pass
 
     def update_special_phrases(self, phrases, should_replace):

test/python/mocks.py

@@ -1,7 +1,9 @@
 """
 Custom mocks for testing.
 """
+import itertools
 
+import psycopg2.extras
 
 class MockParamCapture:
     """ Mock that records the parameters with which a function was called
@@ -16,3 +18,110 @@ class MockParamCapture:
         self.last_args = args
         self.last_kwargs = kwargs
         return self.return_value
+
+
+class MockWordTable:
+    """ A word table for testing.
+    """
+    def __init__(self, conn):
+        self.conn = conn
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE word (word_id INTEGER,
+                                              word_token text,
+                                              word text,
+                                              class text,
+                                              type text,
+                                              country_code varchar(2),
+                                              search_name_count INTEGER,
+                                              operator TEXT)""")
+
+        conn.commit()
+
+    def add_special(self, word_token, word, cls, typ, op):
+        with self.conn.cursor() as cur:
+            cur.execute("""INSERT INTO word (word_token, word, class, type, operator)
+                              VALUES (%s, %s, %s, %s, %s)
+                        """, (word_token, word, cls, typ, op))
+        self.conn.commit()
+
+
+    def add_postcode(self, word_token, postcode):
+        with self.conn.cursor() as cur:
+            cur.execute("""INSERT INTO word (word_token, word, class, type)
+                              VALUES (%s, %s, 'place', 'postcode')
+                        """, (word_token, postcode))
+        self.conn.commit()
+
+
+    def count(self):
+        with self.conn.cursor() as cur:
+            return cur.scalar("SELECT count(*) FROM word")
+
+
+    def count_special(self):
+        with self.conn.cursor() as cur:
+            return cur.scalar("SELECT count(*) FROM word WHERE class != 'place'")
+
+
+    def get_special(self):
+        with self.conn.cursor() as cur:
+            cur.execute("""SELECT word_token, word, class, type, operator
+                           FROM word WHERE class != 'place'""")
+            return set((tuple(row) for row in cur))
+
+
+    def get_postcodes(self):
+        with self.conn.cursor() as cur:
+            cur.execute("""SELECT word FROM word
+                           WHERE class = 'place' and type = 'postcode'""")
+            return set((row[0] for row in cur))
+
+
+class MockPlacexTable:
+    """ A placex table for testing.
+    """
+    def __init__(self, conn):
+        self.idseq = itertools.count(10000)
+        self.conn = conn
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE placex (
+                               place_id BIGINT,
+                               parent_place_id BIGINT,
+                               linked_place_id BIGINT,
+                               importance FLOAT,
+                               indexed_date TIMESTAMP,
+                               geometry_sector INTEGER,
+                               rank_address SMALLINT,
+                               rank_search SMALLINT,
+                               partition SMALLINT,
+                               indexed_status SMALLINT,
+                               osm_id int8,
+                               osm_type char(1),
+                               class text,
+                               type text,
+                               name hstore,
+                               admin_level smallint,
+                               address hstore,
+                               extratags hstore,
+                               geometry Geometry(Geometry,4326),
+                               wikipedia TEXT,
+                               country_code varchar(2),
+                               housenumber TEXT,
+                               postcode TEXT,
+                               centroid GEOMETRY(Geometry, 4326))""")
+            cur.execute("CREATE SEQUENCE IF NOT EXISTS seq_place")
+        conn.commit()
+
+    def add(self, osm_type='N', osm_id=None, cls='amenity', typ='cafe', names=None,
+            admin_level=None, address=None, extratags=None, geom='POINT(10 4)',
+            country=None):
+        with self.conn.cursor() as cur:
+            psycopg2.extras.register_hstore(cur)
+            cur.execute("""INSERT INTO placex (place_id, osm_type, osm_id, class,
+                                               type, name, admin_level, address,
+                                               extratags, geometry, country_code)
+                            VALUES(nextval('seq_place'), %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
+                        (osm_type, osm_id or next(self.idseq), cls, typ, names,
+                         admin_level, address, extratags, 'SRID=4326;' + geom,
+                         country))
+        self.conn.commit()

test/python/test_cli.py

@@ -120,7 +120,7 @@ def test_import_full(temp_db, mock_func_factory, tokenizer_mock):
         mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
         mock_func_factory(nominatim.tools.database_import, 'create_country_names'),
         mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'),
-        mock_func_factory(nominatim.tools.postcodes, 'import_postcodes'),
+        mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
         mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
         mock_func_factory(nominatim.tools.refresh, 'setup_website'),
         mock_func_factory(nominatim.db.properties, 'set_property')
@@ -143,7 +143,7 @@ def test_import_continue_load_data(temp_db, mock_func_factory, tokenizer_mock):
         mock_func_factory(nominatim.tools.database_import, 'load_data'),
         mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
         mock_func_factory(nominatim.tools.database_import, 'create_country_names'),
-        mock_func_factory(nominatim.tools.postcodes, 'import_postcodes'),
+        mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
         mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
         mock_func_factory(nominatim.tools.refresh, 'setup_website'),
         mock_func_factory(nominatim.db.properties, 'set_property')
@@ -280,20 +280,26 @@ def test_special_phrases_csv_command(temp_db, mock_func_factory, tokenizer_mock,
     assert func.called == 1
 
 @pytest.mark.parametrize("command,func", [
-                         ('postcodes', 'update_postcodes'),
                          ('word-counts', 'recompute_word_counts'),
                          ('address-levels', 'load_address_levels_from_file'),
                          ('wiki-data', 'import_wikipedia_articles'),
                          ('importance', 'recompute_importance'),
                          ('website', 'setup_website'),
                          ])
-def test_refresh_command(mock_func_factory, temp_db, command, func):
+def test_refresh_command(mock_func_factory, temp_db, command, func, tokenizer_mock):
     func_mock = mock_func_factory(nominatim.tools.refresh, func)
 
     assert 0 == call_nominatim('refresh', '--' + command)
     assert func_mock.called == 1
 
 
+def test_refresh_postcodes(mock_func_factory, temp_db, tokenizer_mock):
+    func_mock = mock_func_factory(nominatim.tools.postcodes, 'update_postcodes')
+    idx_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_postcodes')
+
+    assert 0 == call_nominatim('refresh', '--postcodes')
+    assert func_mock.called == 1
+
 def test_refresh_create_functions(mock_func_factory, temp_db, tokenizer_mock):
     func_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
 
@@ -302,7 +308,7 @@ def test_refresh_create_functions(mock_func_factory, temp_db, tokenizer_mock):
     assert tokenizer_mock.update_sql_functions_called
 
 
-def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db):
+def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db, tokenizer_mock):
     calls = []
     monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
                         lambda *args, **kwargs: calls.append('import') or 0)

test/python/test_db_async_connection.py

@@ -56,13 +56,21 @@ def test_bad_query(conn):
         conn.wait()
 
 
+def test_bad_query_ignore(temp_db):
+    with closing(DBConnection('dbname=' + temp_db, ignore_sql_errors=True)) as conn:
+        conn.connect()
+
+        conn.perform('SELECT efasfjsea')
+
+        conn.wait()
+
+
 def exec_with_deadlock(cur, sql, detector):
     with DeadlockHandler(lambda *args: detector.append(1)):
         cur.execute(sql)
 
 
 def test_deadlock(simple_conns):
-    print(psycopg2.__version__)
     cur1, cur2 = simple_conns
 
     cur1.execute("""CREATE TABLE t1 (id INT PRIMARY KEY, t TEXT);

test/python/test_tokenizer_legacy.py

@@ -77,12 +77,12 @@ def make_standard_name(temp_db_cursor):
 
 
 @pytest.fixture
-def create_postcode_id(table_factory, temp_db_cursor):
-    table_factory('out_postcode_table', 'postcode TEXT')
-
+def create_postcode_id(temp_db_cursor):
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION create_postcode_id(postcode TEXT)
                               RETURNS BOOLEAN AS $$
-                              INSERT INTO out_postcode_table VALUES (postcode) RETURNING True;
+                              INSERT INTO word (word_token, word, class, type)
+                                VALUES (' ' || postcode, postcode, 'place', 'postcode')
+                              RETURNING True;
                               $$ LANGUAGE SQL""")
 
 
@@ -192,27 +192,38 @@ def test_normalize(analyzer):
     assert analyzer.normalize('TEsT') == 'test'
 
 
-def test_add_postcodes_from_db(analyzer, table_factory, temp_db_cursor,
-                               create_postcode_id):
+def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table,
+                                        create_postcode_id):
     table_factory('location_postcode', 'postcode TEXT',
                   content=(('1234',), ('12 34',), ('AB23',), ('1234',)))
 
-    analyzer.add_postcodes_from_db()
+    analyzer.update_postcodes_from_db()
 
-    assert temp_db_cursor.row_set("SELECT * from out_postcode_table") \
-               == set((('1234', ), ('12 34', ), ('AB23',)))
+    assert word_table.count() == 3
+    assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'}
 
 
-def test_update_special_phrase_empty_table(analyzer, word_table, temp_db_cursor,
-                                           make_standard_name):
+def test_update_postcodes_from_db_add_and_remove(analyzer, table_factory, word_table,
+                                                 create_postcode_id):
+    table_factory('location_postcode', 'postcode TEXT',
+                  content=(('1234',), ('45BC', ), ('XX45', )))
+    word_table.add_postcode(' 1234', '1234')
+    word_table.add_postcode(' 5678', '5678')
+
+    analyzer.update_postcodes_from_db()
+
+    assert word_table.count() == 3
+    assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'}
+
+
+def test_update_special_phrase_empty_table(analyzer, word_table, make_standard_name):
     analyzer.update_special_phrases([
         ("König bei", "amenity", "royal", "near"),
         ("Könige", "amenity", "royal", "-"),
         ("strasse", "highway", "primary", "in")
     ], True)
 
-    assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator
-                                     FROM word WHERE class != 'place'""") \
+    assert word_table.get_special() \
                == set(((' könig bei', 'könig bei', 'amenity', 'royal', 'near'),
                        (' könige', 'könige', 'amenity', 'royal', None),
                        (' strasse', 'strasse', 'highway', 'primary', 'in')))
@@ -220,15 +231,14 @@ def test_update_special_phrase_empty_table(analyzer, word_table, temp_db_cursor,
 
 def test_update_special_phrase_delete_all(analyzer, word_table, temp_db_cursor,
                                           make_standard_name):
-    temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator)
-                              VALUES (' foo', 'foo', 'amenity', 'prison', 'in'),
-                                     (' bar', 'bar', 'highway', 'road', null)""")
+    word_table.add_special(' foo', 'foo', 'amenity', 'prison', 'in')
+    word_table.add_special(' bar', 'bar', 'highway', 'road', None)
 
-    assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
+    assert word_table.count_special() == 2
 
     analyzer.update_special_phrases([], True)
 
-    assert 0 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
+    assert word_table.count_special() == 0
 
 
 def test_update_special_phrases_no_replace(analyzer, word_table, temp_db_cursor,
@@ -244,13 +254,11 @@ def test_update_special_phrases_no_replace(analyzer, word_table, temp_db_cursor,
     assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
 
 
-def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor,
-                                      make_standard_name):
-    temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator)
-                              VALUES (' foo', 'foo', 'amenity', 'prison', 'in'),
-                                     (' bar', 'bar', 'highway', 'road', null)""")
+def test_update_special_phrase_modify(analyzer, word_table, make_standard_name):
+    word_table.add_special(' foo', 'foo', 'amenity', 'prison', 'in')
+    word_table.add_special(' bar', 'bar', 'highway', 'road', None)
 
-    assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
+    assert word_table.count_special() == 2
 
     analyzer.update_special_phrases([
       ('prison', 'amenity', 'prison', 'in'),
@@ -258,8 +266,7 @@ def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor,
       ('garden', 'leisure', 'garden', 'near')
     ], True)
 
-    assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator
-                                     FROM word WHERE class != 'place'""") \
+    assert word_table.get_special() \
                == set(((' prison', 'prison', 'amenity', 'prison', 'in'),
                        (' bar', 'bar', 'highway', 'road', None),
                        (' garden', 'garden', 'leisure', 'garden', 'near')))
@@ -273,21 +280,17 @@ def test_process_place_names(analyzer, make_keywords):
 
 
 @pytest.mark.parametrize('pc', ['12345', 'AB 123', '34-345'])
-def test_process_place_postcode(analyzer, temp_db_cursor, create_postcode_id, pc):
-
+def test_process_place_postcode(analyzer, create_postcode_id, word_table, pc):
     info = analyzer.process_place({'address': {'postcode' : pc}})
 
-    assert temp_db_cursor.row_set("SELECT * from out_postcode_table") \
-               == set(((pc, ),))
+    assert word_table.get_postcodes() == {pc, }
 
 
 @pytest.mark.parametrize('pc', ['12:23', 'ab;cd;f', '123;836'])
-def test_process_place_bad_postcode(analyzer, temp_db_cursor, create_postcode_id,
-                                    pc):
-
+def test_process_place_bad_postcode(analyzer, create_postcode_id, word_table, pc):
     info = analyzer.process_place({'address': {'postcode' : pc}})
 
-    assert 0 == temp_db_cursor.scalar("SELECT count(*) from out_postcode_table")
+    assert not word_table.get_postcodes()
 
 
 @pytest.mark.parametrize('hnr', ['123a', '1', '101'])

test/python/test_tokenizer_legacy_icu.py

@@ -141,16 +141,28 @@ def test_make_standard_hnr(analyzer):
         assert a._make_standard_hnr('iv') == 'IV'
 
 
-def test_add_postcodes_from_db(analyzer, word_table, table_factory, temp_db_cursor):
+def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table):
     table_factory('location_postcode', 'postcode TEXT',
                   content=(('1234',), ('12 34',), ('AB23',), ('1234',)))
 
     with analyzer() as a:
-        a.add_postcodes_from_db()
+        a.update_postcodes_from_db()
 
-    assert temp_db_cursor.row_set("""SELECT word, word_token from word
-                                     """) \
-               == set((('1234', ' 1234'), ('12 34', ' 12 34'), ('AB23', ' AB23')))
+    assert word_table.count() == 3
+    assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'}
+
+
+def test_update_postcodes_from_db_add_and_remove(analyzer, table_factory, word_table):
+    table_factory('location_postcode', 'postcode TEXT',
+                  content=(('1234',), ('45BC', ), ('XX45', )))
+    word_table.add_postcode(' 1234', '1234')
+    word_table.add_postcode(' 5678', '5678')
+
+    with analyzer() as a:
+        a.update_postcodes_from_db()
+
+    assert word_table.count() == 3
+    assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'}
 
 
 def test_update_special_phrase_empty_table(analyzer, word_table, temp_db_cursor):
@@ -224,22 +236,19 @@ def test_process_place_names(analyzer, getorcreate_term_id):
 
 
 @pytest.mark.parametrize('pc', ['12345', 'AB 123', '34-345'])
-def test_process_place_postcode(analyzer, temp_db_cursor, pc):
+def test_process_place_postcode(analyzer, word_table, pc):
     with analyzer() as a:
         info = a.process_place({'address': {'postcode' : pc}})
 
-    assert temp_db_cursor.row_set("""SELECT word FROM word
-                                     WHERE class = 'place' and type = 'postcode'""") \
-               == set(((pc, ),))
+    assert word_table.get_postcodes() == {pc, }
 
 
 @pytest.mark.parametrize('pc', ['12:23', 'ab;cd;f', '123;836'])
-def test_process_place_bad_postcode(analyzer, temp_db_cursor, pc):
+def test_process_place_bad_postcode(analyzer, word_table, pc):
     with analyzer() as a:
         info = a.process_place({'address': {'postcode' : pc}})
 
-    assert 0 == temp_db_cursor.scalar("""SELECT count(*) FROM word
-                                         WHERE class = 'place' and type = 'postcode'""")
+    assert not word_table.get_postcodes()
 
 
 @pytest.mark.parametrize('hnr', ['123a', '1', '101'])

test/python/test_tools_database_import.py

@@ -153,8 +153,8 @@ def test_truncate_database_tables(temp_db_conn, temp_db_cursor, table_factory):
 
 
 @pytest.mark.parametrize("threads", (1, 5))
-def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table, word_table,
-                   temp_db_cursor, threads):
+def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table,
+                   word_table, temp_db_cursor, threads):
     for func in ('precompute_words', 'getorcreate_housenumber_id', 'make_standard_name'):
         temp_db_cursor.execute("""CREATE FUNCTION {} (src TEXT)
                                   RETURNS TEXT AS $$ SELECT 'a'::TEXT $$ LANGUAGE SQL

test/python/test_tools_postcodes.py

@@ -1,55 +1,185 @@
 """
 Tests for functions to maintain the artificial postcode table.
 """
+import subprocess
 
 import pytest
 
 from nominatim.tools import postcodes
 import dummy_tokenizer
 
+class MockPostcodeTable:
+    """ A location_postcode table for testing.
+    """
+    def __init__(self, conn):
+        self.conn = conn
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE location_postcode (
+                               place_id BIGINT,
+                               parent_place_id BIGINT,
+                               rank_search SMALLINT,
+                               rank_address SMALLINT,
+                               indexed_status SMALLINT,
+                               indexed_date TIMESTAMP,
+                               country_code varchar(2),
+                               postcode TEXT,
+                               geometry GEOMETRY(Geometry, 4326))""")
+            cur.execute("""CREATE OR REPLACE FUNCTION token_normalized_postcode(postcode TEXT)
+                           RETURNS TEXT AS $$ BEGIN RETURN postcode; END; $$ LANGUAGE plpgsql;
+                        """)
+        conn.commit()
+
+    def add(self, country, postcode, x, y):
+        with self.conn.cursor() as cur:
+            cur.execute("""INSERT INTO location_postcode (place_id, indexed_status,
+                                                          country_code, postcode,
+                                                          geometry)
+                           VALUES (nextval('seq_place'), 1, %s, %s,
+                                   'SRID=4326;POINT(%s %s)')""",
+                        (country, postcode, x, y))
+        self.conn.commit()
+
+
+    @property
+    def row_set(self):
+        with self.conn.cursor() as cur:
+            cur.execute("""SELECT country_code, postcode,
+                                  ST_X(geometry), ST_Y(geometry)
+                           FROM location_postcode""")
+            return set((tuple(row) for row in cur))
+
+
 @pytest.fixture
 def tokenizer():
     return dummy_tokenizer.DummyTokenizer(None, None)
 
 @pytest.fixture
-def postcode_table(temp_db_with_extensions, temp_db_cursor, table_factory,
-                   placex_table, word_table):
-    table_factory('location_postcode',
-                  """ place_id BIGINT,
-                      parent_place_id BIGINT,
-                      rank_search SMALLINT,
-                      rank_address SMALLINT,
-                      indexed_status SMALLINT,
-                      indexed_date TIMESTAMP,
-                      country_code varchar(2),
-                      postcode TEXT,
-                      geometry GEOMETRY(Geometry, 4326)""")
-    temp_db_cursor.execute('CREATE SEQUENCE seq_place')
-    temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION token_normalized_postcode(postcode TEXT)
-                              RETURNS TEXT AS $$ BEGIN RETURN postcode; END; $$ LANGUAGE plpgsql;
-                           """)
+def postcode_table(temp_db_conn, placex_table, word_table):
+    return MockPostcodeTable(temp_db_conn)
 
 
-def test_import_postcodes_empty(dsn, temp_db_cursor, postcode_table, tmp_path, tokenizer):
-    postcodes.import_postcodes(dsn, tmp_path, tokenizer)
+def test_import_postcodes_empty(dsn, postcode_table, tmp_path, tokenizer):
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
 
-    assert temp_db_cursor.table_exists('gb_postcode')
-    assert temp_db_cursor.table_exists('us_postcode')
-    assert temp_db_cursor.table_rows('location_postcode') == 0
+    assert not postcode_table.row_set
 
 
-def test_import_postcodes_from_placex(dsn, temp_db_cursor, postcode_table, tmp_path, tokenizer):
-    temp_db_cursor.execute("""
-        INSERT INTO placex (place_id, country_code, address, geometry)
-          VALUES (1, 'xx', '"postcode"=>"9486"', 'SRID=4326;POINT(10 12)')
-    """)
+def test_import_postcodes_add_new(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='9486'))
+    postcode_table.add('yy', '9486', 99, 34)
 
-    postcodes.import_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
 
-    rows = temp_db_cursor.row_set(""" SELECT postcode, country_code,
-                                      ST_X(geometry), ST_Y(geometry)
-                                      FROM location_postcode""")
-    print(rows)
-    assert len(rows) == 1
-    assert rows == set((('9486', 'xx', 10, 12), ))
+    assert postcode_table.row_set == {('xx', '9486', 10, 12), }
 
+
+def test_import_postcodes_replace_coordinates(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+    postcode_table.add('xx', 'AB 4511', 99, 34)
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
+
+
+def test_import_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+    postcode_table.add('xx', 'AB 4511', 10, 11.99999)
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('xx', 'AB 4511', 10, 11.99999)}
+
+
+def test_import_postcodes_remove(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+    postcode_table.add('xx', 'badname', 10, 12)
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
+
+
+def test_import_postcodes_ignore_empty_country(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    placex_table.add(country=None, geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert not postcode_table.row_set
+
+
+def test_import_postcodes_remove_all(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    postcode_table.add('ch', '5613', 10, 12)
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert not postcode_table.row_set
+
+
+def test_import_postcodes_multi_country(dsn, placex_table, postcode_table, tmp_path, tokenizer):
+    placex_table.add(country='de', geom='POINT(10 12)',
+                     address=dict(postcode='54451'))
+    placex_table.add(country='cc', geom='POINT(100 56)',
+                     address=dict(postcode='DD23 T'))
+    placex_table.add(country='de', geom='POINT(10.3 11.0)',
+                     address=dict(postcode='54452'))
+    placex_table.add(country='cc', geom='POINT(10.3 11.0)',
+                     address=dict(postcode='54452'))
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('de', '54451', 10, 12),
+                                      ('de', '54452', 10.3, 11.0),
+                                      ('cc', '54452', 10.3, 11.0),
+                                      ('cc', 'DD23 T', 100, 56)}
+
+
+@pytest.mark.parametrize("gzipped", [True, False])
+def test_import_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
+                                 tokenizer, gzipped):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+
+    extfile = tmp_path / 'xx_postcodes.csv'
+    extfile.write_text("postcode,lat,lon\nAB 4511,-4,-1\nCD 4511,-5, -10")
+
+    if gzipped:
+        subprocess.run(['gzip', str(extfile)])
+        assert not extfile.is_file()
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12),
+                                      ('xx', 'CD 4511', -10, -5)}
+
+
+def test_import_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
+                                            tmp_path, tokenizer):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+
+    extfile = tmp_path / 'xx_postcodes.csv'
+    extfile.write_text("postode,lat,lon\nAB 4511,-4,-1\nCD 4511,-5, -10")
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
+
+
+def test_import_postcodes_extern_bad_number(dsn, placex_table, postcode_table,
+                                            tmp_path, tokenizer):
+    placex_table.add(country='xx', geom='POINT(10 12)',
+                     address=dict(postcode='AB 4511'))
+
+    extfile = tmp_path / 'xx_postcodes.csv'
+    extfile.write_text("postcode,lat,lon\nXX 4511,-4,NaN\nCD 4511,-5, -10\n34,200,0")
+
+    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+
+    assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12),
+                                      ('xx', 'CD 4511', -10, -5)}

test/python/test_tools_refresh_setup_website.py

@@ -18,16 +18,16 @@ def envdir(tmpdir):
 @pytest.fixture
 def test_script(envdir):
     def _create_file(code):
-        outfile = envdir / 'php' / 'website' / 'search.php'
+        outfile = envdir / 'php' / 'website' / 'reverse-only-search.php'
         outfile.write_text('<?php\n{}\n'.format(code), 'utf-8')
 
     return _create_file
 
 
-def run_website_script(envdir, config):
+def run_website_script(envdir, config, conn):
     config.lib_dir.php = envdir / 'php'
     config.project_dir = envdir
-    refresh.setup_website(envdir, config)
+    refresh.setup_website(envdir, config, conn)
 
     proc = subprocess.run(['/usr/bin/env', 'php', '-Cq',
                            envdir / 'search.php'], check=False)
@@ -37,36 +37,39 @@ def run_website_script(envdir, config):
 
 @pytest.mark.parametrize("setting,retval", (('yes', 10), ('no', 20)))
 def test_setup_website_check_bool(def_config, monkeypatch, envdir, test_script,
-                                  setting, retval):
+                                  setting, retval, temp_db_conn):
     monkeypatch.setenv('NOMINATIM_CORS_NOACCESSCONTROL', setting)
 
     test_script('exit(CONST_NoAccessControl ? 10 : 20);')
 
-    assert run_website_script(envdir, def_config) == retval
+    assert run_website_script(envdir, def_config, temp_db_conn) == retval
 
 
 @pytest.mark.parametrize("setting", (0, 10, 99067))
-def test_setup_website_check_int(def_config, monkeypatch, envdir, test_script, setting):
+def test_setup_website_check_int(def_config, monkeypatch, envdir, test_script, setting,
+                                 temp_db_conn):
     monkeypatch.setenv('NOMINATIM_LOOKUP_MAX_COUNT', str(setting))
 
     test_script('exit(CONST_Places_Max_ID_count == {} ? 10 : 20);'.format(setting))
 
-    assert run_website_script(envdir, def_config) == 10
+    assert run_website_script(envdir, def_config, temp_db_conn) == 10
 
 
-def test_setup_website_check_empty_str(def_config, monkeypatch, envdir, test_script):
+def test_setup_website_check_empty_str(def_config, monkeypatch, envdir, test_script,
+                                       temp_db_conn):
     monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', '')
 
     test_script('exit(CONST_Default_Language === false ? 10 : 20);')
 
-    assert run_website_script(envdir, def_config) == 10
+    assert run_website_script(envdir, def_config, temp_db_conn) == 10
 
 
-def test_setup_website_check_str(def_config, monkeypatch, envdir, test_script):
+def test_setup_website_check_str(def_config, monkeypatch, envdir, test_script,
+                                 temp_db_conn):
     monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'ffde 2')
 
     test_script('exit(CONST_Default_Language === "ffde 2" ? 10 : 20);')
 
-    assert run_website_script(envdir, def_config) == 10
+    assert run_website_script(envdir, def_config, temp_db_conn) == 10
 
 

test/python/test_tools_tiger_data.py

@@ -2,60 +2,137 @@
 Test for tiger data function
 """
 from pathlib import Path
+from textwrap import dedent
 
 import pytest
 import tarfile
 
 from nominatim.tools import tiger_data, database_import
+from nominatim.errors import UsageError
+
+class MockTigerTable:
+
+    def __init__(self, conn):
+        self.conn = conn
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE tiger (linegeo GEOMETRY,
+                                               start INTEGER,
+                                               stop INTEGER,
+                                               interpol TEXT,
+                                               token_info JSONB,
+                                               postcode TEXT)""")
+
+    def count(self):
+        with self.conn.cursor() as cur:
+            return cur.scalar("SELECT count(*) FROM tiger")
+
+    def row(self):
+        with self.conn.cursor() as cur:
+            cur.execute("SELECT * FROM tiger LIMIT 1")
+            return cur.fetchone()
+
+@pytest.fixture
+def tiger_table(def_config, temp_db_conn, sql_preprocessor,
+                temp_db_with_extensions, tmp_path):
+    def_config.lib_dir.sql = tmp_path / 'sql'
+    def_config.lib_dir.sql.mkdir()
+
+    (def_config.lib_dir.sql / 'tiger_import_start.sql').write_text(
+        """CREATE OR REPLACE FUNCTION tiger_line_import(linegeo GEOMETRY, start INTEGER,
+                                                        stop INTEGER, interpol TEXT,
+                                                        token_info JSONB, postcode TEXT)
+           RETURNS INTEGER AS $$
+            INSERT INTO tiger VALUES(linegeo, start, stop, interpol, token_info, postcode) RETURNING 1
+           $$ LANGUAGE SQL;""")
+    (def_config.lib_dir.sql / 'tiger_import_finish.sql').write_text(
+        """DROP FUNCTION tiger_line_import (linegeo GEOMETRY, in_startnumber INTEGER,
+                                 in_endnumber INTEGER, interpolationtype TEXT,
+                                 token_info JSONB, in_postcode TEXT);""")
+
+    return MockTigerTable(temp_db_conn)
+
+
+@pytest.fixture
+def csv_factory(tmp_path):
+    def _mk_file(fname, hnr_from=1, hnr_to=9, interpol='odd', street='Main St',
+                 city='Newtown', state='AL', postcode='12345',
+                 geometry='LINESTRING(-86.466995 32.428956,-86.466923 32.428933)'):
+        (tmp_path / (fname + '.csv')).write_text(dedent("""\
+        from;to;interpolation;street;city;state;postcode;geometry
+        {};{};{};{};{};{};{};{}
+        """.format(hnr_from, hnr_to, interpol, street, city, state,
+                   postcode, geometry)))
+
+    return _mk_file
 
 
 @pytest.mark.parametrize("threads", (1, 5))
-def test_add_tiger_data(def_config, tmp_path, sql_preprocessor,
-                        temp_db_cursor, threads, temp_db_with_extensions):
-    temp_db_cursor.execute('CREATE TABLE place (id INT)')
-    sqlfile = tmp_path / '1010.sql'
-    sqlfile.write_text("""INSERT INTO place values (1);
-                          INSERT INTO non_existant_table values (1);""")
-    tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
+def test_add_tiger_data(def_config, src_dir, tiger_table, tokenizer_mock, threads):
+    tiger_data.add_tiger_data(str(src_dir / 'test' / 'testdb' / 'tiger'),
+                              def_config, threads, tokenizer_mock())
 
-    assert temp_db_cursor.table_rows('place') == 1
+    assert tiger_table.count() == 6213
 
 
-@pytest.mark.parametrize("threads", (1, 5))
-def test_add_tiger_data_bad_file(def_config, tmp_path, sql_preprocessor,
-                                 temp_db_cursor, threads, temp_db_with_extensions):
-    temp_db_cursor.execute('CREATE TABLE place (id INT)')
-    sqlfile = tmp_path / '1010.txt'
+def test_add_tiger_data_no_files(def_config, tiger_table, tokenizer_mock,
+                                 tmp_path):
+    tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())
+
+    assert tiger_table.count() == 0
+
+
+def test_add_tiger_data_bad_file(def_config, tiger_table, tokenizer_mock,
+                                 tmp_path):
+    sqlfile = tmp_path / '1010.csv'
     sqlfile.write_text("""Random text""")
-    tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
 
-    assert temp_db_cursor.table_rows('place') == 0
+    tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())
+
+    assert tiger_table.count() == 0
+
+
+def test_add_tiger_data_hnr_nan(def_config, tiger_table, tokenizer_mock,
+                                csv_factory, tmp_path):
+    csv_factory('file1', hnr_from=99)
+    csv_factory('file2', hnr_from='L12')
+    csv_factory('file3', hnr_to='12.4')
+
+    tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())
+
+    assert tiger_table.count() == 1
+    assert tiger_table.row()['start'] == 99
 
 
 @pytest.mark.parametrize("threads", (1, 5))
-def test_add_tiger_data_tarfile(def_config, tmp_path, temp_db_cursor,
-                                threads, temp_db_with_extensions, sql_preprocessor):
-    temp_db_cursor.execute('CREATE TABLE place (id INT)')
-    sqlfile = tmp_path / '1010.sql'
-    sqlfile.write_text("""INSERT INTO place values (1);
-                          INSERT INTO non_existant_table values (1);""")
+def test_add_tiger_data_tarfile(def_config, tiger_table, tokenizer_mock,
+                                tmp_path, src_dir, threads):
     tar = tarfile.open(str(tmp_path / 'sample.tar.gz'), "w:gz")
-    tar.add(sqlfile)
+    tar.add(str(src_dir / 'test' / 'testdb' / 'tiger' / '01001.csv'))
     tar.close()
-    tiger_data.add_tiger_data(str(tmp_path / 'sample.tar.gz'), def_config, threads)
 
-    assert temp_db_cursor.table_rows('place') == 1
+    tiger_data.add_tiger_data(str(tmp_path / 'sample.tar.gz'), def_config, 1,
+                              tokenizer_mock())
+
+    assert tiger_table.count() == 6213
 
 
-@pytest.mark.parametrize("threads", (1, 5))
-def test_add_tiger_data_bad_tarfile(def_config, tmp_path, temp_db_cursor, threads,
-                                    temp_db_with_extensions, sql_preprocessor):
-    temp_db_cursor.execute('CREATE TABLE place (id INT)')
-    sqlfile = tmp_path / '1010.txt'
-    sqlfile.write_text("""Random text""")
+def test_add_tiger_data_bad_tarfile(def_config, tiger_table, tokenizer_mock,
+                                    tmp_path):
+    tarfile = tmp_path / 'sample.tar.gz'
+    tarfile.write_text("""Random text""")
+
+    with pytest.raises(UsageError):
+        tiger_data.add_tiger_data(str(tarfile), def_config, 1, tokenizer_mock())
+
+
+def test_add_tiger_data_empty_tarfile(def_config, tiger_table, tokenizer_mock,
+                                      tmp_path, src_dir):
     tar = tarfile.open(str(tmp_path / 'sample.tar.gz'), "w:gz")
-    tar.add(sqlfile)
+    tar.add(__file__)
     tar.close()
-    tiger_data.add_tiger_data(str(tmp_path / 'sample.tar.gz'), def_config, threads)
 
-    assert temp_db_cursor.table_rows('place') == 0
+    tiger_data.add_tiger_data(str(tmp_path / 'sample.tar.gz'), def_config, 1,
+                              tokenizer_mock())
+
+    assert tiger_table.count() == 0
+