Add tests for the new SPWikiLoader and SPCsvLoader

test/python/test_cli.py

@@ -255,13 +255,21 @@ def test_index_command(mock_func_factory, temp_db_cursor, tokenizer_mock,
     assert bnd_mock.called == do_bnds
     assert rank_mock.called == do_ranks
 
-def test_special_phrases_command(temp_db, mock_func_factory, tokenizer_mock):
+def test_special_phrases_wiki_command(temp_db, mock_func_factory, tokenizer_mock):
     func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
 
     call_nominatim('special-phrases', '--import-from-wiki')
 
     assert func.called == 1
 
+def test_special_phrases_csv_command(temp_db, mock_func_factory, tokenizer_mock):
+    func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
+    testdata = Path('__file__') / '..' / '..' / 'testdb'
+    csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
+    call_nominatim('special-phrases', '--import-from-csv', csv_path)
+
+    assert func.called == 1
+
 @pytest.mark.parametrize("command,func", [
                          ('postcodes', 'update_postcodes'),
                          ('word-counts', 'recompute_word_counts'),

test/python/test_tools_import_special_phrases.py

@@ -185,7 +185,7 @@ def test_remove_non_existent_tables_from_db(sp_importer, default_phrases,
             tables_result[0][0] == 'place_classtype_testclasstypetable_to_keep'
         )
 
-def test_import_from_wiki(monkeypatch, temp_db_conn, def_config, sp_importer,
+def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
                           placex_table, tokenizer_mock):
     """
         Check that the main import_phrases() method is well executed.

test/python/test_tools_sp_csv_loader.py

@@ -0,0 +1,59 @@
+"""
+    Tests for methods of the SPCsvLoader class.
+"""
+from nominatim.errors import UsageError
+import pytest
+from pathlib import Path
+from nominatim.tools.special_phrases.sp_csv_loader import SPCsvLoader
+
+TEST_BASE_DIR = Path(__file__) / '..' / '..'
+
+def test_parse_csv(sp_csv_loader):
+    """
+        Test method parse_csv()
+        Should return the right SpecialPhrase objects.
+    """
+    phrases = sp_csv_loader.parse_csv(sp_csv_loader.csv_path)
+    assert check_phrases_content(phrases)
+
+
+def test_next(sp_csv_loader):
+    """
+        Test objects returned from the next() method.
+        It should return all SpecialPhrases objects of
+        the sp_csv_test.csv special phrases.
+    """
+    phrases = next(sp_csv_loader)
+    assert check_phrases_content(phrases)
+
+def test_check_csv_validity(sp_csv_loader):
+    """
+        Test method check_csv_validity()
+        It should raise an exception when file with a
+        different exception than .csv is given.
+    """
+    sp_csv_loader.csv_path = 'test.csv'
+    sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
+    sp_csv_loader.csv_path = 'test.wrong'
+    with pytest.raises(UsageError):
+        assert sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
+
+def check_phrases_content(phrases):
+    """
+        Asserts that the given phrases list contains
+        the right phrases of the sp_csv_test.csv special phrases.
+    """
+    return  len(phrases) > 1 \
+            and any(p.p_label == 'Billboard' and p.p_class == 'advertising' and p.p_type == 'billboard'
+                    and p.p_operator == '-' for p in phrases) \
+            and any(p.p_label == 'Zip Lines' and p.p_class == 'aerialway' and p.p_type == 'zip_line'
+                    and p.p_operator == '-' for p in phrases)
+
+@pytest.fixture
+def sp_csv_loader():
+    """
+        Return an instance of SPCsvLoader.
+    """
+    csv_path = (TEST_BASE_DIR / 'testdata' / 'sp_csv_test.csv').resolve()
+    loader = SPCsvLoader(csv_path)
+    return loader

test/python/test_tools_sp_wiki_loader.py

@@ -0,0 +1,63 @@
+"""
+    Tests for methods of the SPWikiLoader class.
+"""
+import pytest
+from pathlib import Path
+from nominatim.tools.special_phrases.sp_wiki_loader import SPWikiLoader
+
+TEST_BASE_DIR = Path(__file__) / '..' / '..'
+
+def test_parse_xml(sp_wiki_loader):
+    """
+        Test method parse_xml()
+        Should return the right SpecialPhrase objects.
+    """
+    xml = get_test_xml_wiki_content()
+    phrases = sp_wiki_loader.parse_xml(xml)
+    assert check_phrases_content(phrases)
+
+
+def test_next(sp_wiki_loader):
+    """
+        Test objects returned from the next() method.
+        It should return all SpecialPhrases objects of
+        the 'en' special phrases.
+    """
+    phrases = next(sp_wiki_loader)
+    assert check_phrases_content(phrases)
+
+def check_phrases_content(phrases):
+    """
+        Asserts that the given phrases list contains
+        the right phrases of the 'en' special phrases.
+    """
+    return  len(phrases) > 1 \
+            and any(p.p_label == 'Embassies' and p.p_class == 'amenity' and p.p_type == 'embassy'
+                    and p.p_operator == '-' for p in phrases) \
+            and any(p.p_label == 'Zip Line' and p.p_class == 'aerialway' and p.p_type == 'zip_line'
+                    and p.p_operator == '-' for p in phrases)
+
+@pytest.fixture
+def sp_wiki_loader(monkeypatch, def_config):
+    """
+        Return an instance of SPWikiLoader.
+    """
+    loader = SPWikiLoader(def_config, ['en'])
+    monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content',
+                        mock_get_wiki_content)
+    return loader
+
+def mock_get_wiki_content(lang):
+    """
+        Mock the _get_wiki_content() method to return
+        static xml test file content.
+    """
+    return get_test_xml_wiki_content()
+
+def get_test_xml_wiki_content():
+    """
+        return the content of the static xml test file.
+    """
+    xml_test_content_path = (TEST_BASE_DIR / 'testdata' / 'special_phrases_test_content.txt').resolve()
+    with open(xml_test_content_path) as xml_content_reader:
+        return xml_content_reader.read()