Add tests for the new SPWikiLoader and SPCsvLoader

test/bdd/steps/nominatim_environment.py

@@ -200,11 +200,8 @@ class NominatimEnvironment:
                         phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
                         run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
                     else:
-                        # XXX Temporary use the wiki while there is no CSV import
+                        csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
-                        # available.
+                        self.run_nominatim('special-phrases', '--import-from-csv', csv_path)
-                        self.test_env['NOMINATIM_LANGUAGES'] = 'en'
-                        self.run_nominatim('special-phrases', '--import-from-wiki')
-                        del self.test_env['NOMINATIM_LANGUAGES']
                 except:
                     self.db_drop_database(self.api_test_db)
                     raise

test/python/test_cli.py

@@ -255,13 +255,21 @@ def test_index_command(mock_func_factory, temp_db_cursor, tokenizer_mock,
     assert bnd_mock.called == do_bnds
     assert rank_mock.called == do_ranks
 
-def test_special_phrases_command(temp_db, mock_func_factory, tokenizer_mock):
+def test_special_phrases_wiki_command(temp_db, mock_func_factory, tokenizer_mock):
     func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
 
     call_nominatim('special-phrases', '--import-from-wiki')
 
     assert func.called == 1
 
+def test_special_phrases_csv_command(temp_db, mock_func_factory, tokenizer_mock):
+    func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
+    testdata = Path('__file__') / '..' / '..' / 'testdb'
+    csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
+    call_nominatim('special-phrases', '--import-from-csv', csv_path)
+
+    assert func.called == 1
+
 @pytest.mark.parametrize("command,func", [
                          ('postcodes', 'update_postcodes'),
                          ('word-counts', 'recompute_word_counts'),

test/python/test_tools_import_special_phrases.py

@@ -185,7 +185,7 @@ def test_remove_non_existent_tables_from_db(sp_importer, default_phrases,
             tables_result[0][0] == 'place_classtype_testclasstypetable_to_keep'
         )
 
-def test_import_from_wiki(monkeypatch, temp_db_conn, def_config, sp_importer,
+def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
                           placex_table, tokenizer_mock):
     """
         Check that the main import_phrases() method is well executed.

test/python/test_tools_sp_csv_loader.py

@@ -0,0 +1,59 @@
+"""
+    Tests for methods of the SPCsvLoader class.
+"""
+from nominatim.errors import UsageError
+import pytest
+from pathlib import Path
+from nominatim.tools.special_phrases.sp_csv_loader import SPCsvLoader
+
+TEST_BASE_DIR = Path(__file__) / '..' / '..'
+
+def test_parse_csv(sp_csv_loader):
+    """
+        Test method parse_csv()
+        Should return the right SpecialPhrase objects.
+    """
+    phrases = sp_csv_loader.parse_csv(sp_csv_loader.csv_path)
+    assert check_phrases_content(phrases)
+
+
+def test_next(sp_csv_loader):
+    """
+        Test objects returned from the next() method.
+        It should return all SpecialPhrases objects of
+        the sp_csv_test.csv special phrases.
+    """
+    phrases = next(sp_csv_loader)
+    assert check_phrases_content(phrases)
+
+def test_check_csv_validity(sp_csv_loader):
+    """
+        Test method check_csv_validity()
+        It should raise an exception when file with a
+        different exception than .csv is given.
+    """
+    sp_csv_loader.csv_path = 'test.csv'
+    sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
+    sp_csv_loader.csv_path = 'test.wrong'
+    with pytest.raises(UsageError):
+        assert sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
+
+def check_phrases_content(phrases):
+    """
+        Asserts that the given phrases list contains
+        the right phrases of the sp_csv_test.csv special phrases.
+    """
+    return  len(phrases) > 1 \
+            and any(p.p_label == 'Billboard' and p.p_class == 'advertising' and p.p_type == 'billboard'
+                    and p.p_operator == '-' for p in phrases) \
+            and any(p.p_label == 'Zip Lines' and p.p_class == 'aerialway' and p.p_type == 'zip_line'
+                    and p.p_operator == '-' for p in phrases)
+
+@pytest.fixture
+def sp_csv_loader():
+    """
+        Return an instance of SPCsvLoader.
+    """
+    csv_path = (TEST_BASE_DIR / 'testdata' / 'sp_csv_test.csv').resolve()
+    loader = SPCsvLoader(csv_path)
+    return loader

test/python/test_tools_sp_wiki_loader.py

@@ -0,0 +1,63 @@
+"""
+    Tests for methods of the SPWikiLoader class.
+"""
+import pytest
+from pathlib import Path
+from nominatim.tools.special_phrases.sp_wiki_loader import SPWikiLoader
+
+TEST_BASE_DIR = Path(__file__) / '..' / '..'
+
+def test_parse_xml(sp_wiki_loader):
+    """
+        Test method parse_xml()
+        Should return the right SpecialPhrase objects.
+    """
+    xml = get_test_xml_wiki_content()
+    phrases = sp_wiki_loader.parse_xml(xml)
+    assert check_phrases_content(phrases)
+
+
+def test_next(sp_wiki_loader):
+    """
+        Test objects returned from the next() method.
+        It should return all SpecialPhrases objects of
+        the 'en' special phrases.
+    """
+    phrases = next(sp_wiki_loader)
+    assert check_phrases_content(phrases)
+
+def check_phrases_content(phrases):
+    """
+        Asserts that the given phrases list contains
+        the right phrases of the 'en' special phrases.
+    """
+    return  len(phrases) > 1 \
+            and any(p.p_label == 'Embassies' and p.p_class == 'amenity' and p.p_type == 'embassy'
+                    and p.p_operator == '-' for p in phrases) \
+            and any(p.p_label == 'Zip Line' and p.p_class == 'aerialway' and p.p_type == 'zip_line'
+                    and p.p_operator == '-' for p in phrases)
+
+@pytest.fixture
+def sp_wiki_loader(monkeypatch, def_config):
+    """
+        Return an instance of SPWikiLoader.
+    """
+    loader = SPWikiLoader(def_config, ['en'])
+    monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content',
+                        mock_get_wiki_content)
+    return loader
+
+def mock_get_wiki_content(lang):
+    """
+        Mock the _get_wiki_content() method to return
+        static xml test file content.
+    """
+    return get_test_xml_wiki_content()
+
+def get_test_xml_wiki_content():
+    """
+        return the content of the static xml test file.
+    """
+    xml_test_content_path = (TEST_BASE_DIR / 'testdata' / 'special_phrases_test_content.txt').resolve()
+    with open(xml_test_content_path) as xml_content_reader:
+        return xml_content_reader.read()

test/testdata/sp_csv_test.csv

@@ -0,0 +1,42 @@
+phrase,class,type,operator,plural 
+Zip Lines,aerialway,zip_line,-,Y 
+Zip Line in,aerialway,zip_line,in,N 
+Zip Lines in,aerialway,zip_line,in,Y 
+Zip Line near,aerialway,zip_line,near,N 
+Zip Lines near,aerialway,zip_line,near,Y 
+Zip Wire,aerialway,zip_line,-,N 
+Zip Wires,aerialway,zip_line,-,Y 
+Zip Wire in,aerialway,zip_line,in,N 
+Zip Wires in,aerialway,zip_line,in,Y 
+Zip Wire near,aerialway,zip_line,near,N 
+Zip Wires near,aerialway,zip_line,near,Y 
+Zipline,aerialway,zip_line,-,N 
+Ziplines,aerialway,zip_line,-,Y 
+Zipline in,aerialway,zip_line,in,N 
+Ziplines in,aerialway,zip_line,in,Y 
+Zipline near,aerialway,zip_line,near,N 
+Ziplines near,aerialway,zip_line,near,Y 
+Zipwire,aerialway,zip_line,-,N 
+Zipwires,aerialway,zip_line,-,Y 
+Zipwire in,aerialway,zip_line,in,N 
+Zipwires in,aerialway,zip_line,in,Y 
+Zipwire near,aerialway,zip_line,near,N 
+Zipwires near,aerialway,zip_line,near,Y 
+Aerodrome,aeroway,aerodrome,-,N 
+Aerodromes,aeroway,aerodrome,-,Y 
+Aerodrome in,aeroway,aerodrome,in,N 
+Aerodromes in,aeroway,aerodrome,in,Y 
+Aerodrome near,aeroway,aerodrome,near,N 
+Aerodromes near,aeroway,aerodrome,near,Y 
+Airport,aeroway,aerodrome,-,N 
+Airports,aeroway,aerodrome,-,Y 
+Airport in,aeroway,aerodrome,in,N 
+Airports in,aeroway,aerodrome,in,Y 
+Airport near,aeroway,aerodrome,near,N 
+Airports near,aeroway,aerodrome,near,Y 
+Billboard,advertising,billboard,-,N 
+Billboards,advertising,billboard,-,Y 
+Billboard in,advertising,billboard,in,N 
+Billboards in,advertising,billboard,in,Y 
+Billboard near,advertising,billboard,near,N 
+Billboards near,advertising,billboard,near,Y