safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

Merge pull request #2760 from lonvia/reorganize-data-classes

Browse Source 
Code cleanup: move some common code into the data submodule
This commit is contained in:
Sarah Hoffmann
2022-07-07 16:12:11 +02:00
committed by GitHub
parent 69e51aebab 856925d19b
commit 300612c5a8
22 changed files with 39 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nominatim/clicmd/setup.py
View File

@@ -63,8 +63,9 @@ class SetupAll:
@staticmethod
def run(args):
from ..tools import database_import, refresh, postcodes, freeze, country_info
def run(args): # pylint: disable=too-many-statements
from ..data import country_info
from ..tools import database_import, refresh, postcodes, freeze
from ..indexer.indexer import Indexer
country_info.setup_country_config(args.config)

0
nominatim/tools/country_info.py → nominatim/data/country_info.py
View File

9
nominatim/indexer/place_info.py → nominatim/data/place_info.py
View File

@@ -9,8 +9,6 @@ Wrapper around place information the indexer gets from the database and hands to
the tokenizer.
"""
import psycopg2.extras
class PlaceInfo:
""" Data class containing all information the tokenizer gets about a
place it should process the names for.
@@ -20,13 +18,6 @@ class PlaceInfo:
self._info = info
def analyze(self, analyzer):
""" Process this place with the given tokenizer and return the
result in psycopg2-compatible Json.
"""
return psycopg2.extras.Json(analyzer.process_place(self))
@property
def name(self):
""" A dictionary with the names of the place or None if the place

2
nominatim/data/postcode_format.py
View File

@@ -11,7 +11,7 @@ format.
import re
from nominatim.errors import UsageError
from nominatim.tools import country_info
from nominatim.data import country_info
class CountryPostcodeMatcher:
""" Matches and formats a postcode according to a format definition

9
nominatim/indexer/runners.py
View File

@@ -11,14 +11,17 @@ tasks.
import functools
from psycopg2 import sql as pysql
import psycopg2.extras
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
# pylint: disable=C0111
def _mk_valuelist(template, num):
return pysql.SQL(',').join([pysql.SQL(template)] * num)
def _analyze_place(place, analyzer):
return psycopg2.extras.Json(analyzer.process_place(PlaceInfo(place)))
class AbstractPlacexRunner:
""" Returns SQL commands for indexing of the placex table.
@@ -56,7 +59,7 @@ class AbstractPlacexRunner:
for place in places:
for field in ('place_id', 'name', 'address', 'linked_place_id'):
values.append(place[field])
values.append(PlaceInfo(place).analyze(self.analyzer))
values.append(_analyze_place(place, self.analyzer))
worker.perform(self._index_sql(len(places)), values)
@@ -150,7 +153,7 @@ class InterpolationRunner:
values = []
for place in places:
values.extend((place[x] for x in ('place_id', 'address')))
values.append(PlaceInfo(place).analyze(self.analyzer))
values.append(_analyze_place(place, self.analyzer))
worker.perform(self._index_sql(len(places)), values)

2
nominatim/tokenizer/base.py
View File

@@ -12,7 +12,7 @@ from abc import ABC, abstractmethod
from typing import List, Tuple, Dict, Any
from nominatim.config import Configuration
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
# pylint: disable=unnecessary-pass

4
nominatim/tokenizer/icu_rule_loader.py
View File

@@ -17,7 +17,7 @@ from nominatim.db.properties import set_property, get_property
from nominatim.errors import UsageError
from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
from nominatim.tokenizer.icu_token_analysis import ICUTokenAnalysis
import nominatim.tools.country_info
import nominatim.data.country_info
LOG = logging.getLogger()
@@ -46,7 +46,7 @@ class ICURuleLoader:
config='TOKENIZER_CONFIG')
# Make sure country information is available to analyzers and sanitizers.
nominatim.tools.country_info.setup_country_config(config)
nominatim.data.country_info.setup_country_config(config)
self.normalization_rules = self._cfg_to_icu_rules(rules, 'normalization')
self.transliteration_rules = self._cfg_to_icu_rules(rules, 'transliteration')

2
nominatim/tokenizer/icu_tokenizer.py
View File

@@ -16,7 +16,7 @@ from textwrap import dedent
from nominatim.db.connection import connect
from nominatim.db.utils import CopyBuffer
from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
from nominatim.tokenizer.icu_rule_loader import ICURuleLoader
from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer

2
nominatim/tokenizer/sanitizers/tag_analyzer_by_language.py
View File

@@ -30,7 +30,7 @@ Arguments:
any analyzer tagged) is retained. (default: replace)
"""
from nominatim.tools import country_info
from nominatim.data import country_info
class _AnalyzerByLanguage:
""" Processor for tagging the language of names in a place.

6
nominatim/tools/tiger_data.py
View File

@@ -13,11 +13,13 @@ import logging
import os
import tarfile
from psycopg2.extras import Json
from nominatim.db.connection import connect
from nominatim.db.async_connection import WorkerPool
from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.errors import UsageError
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
LOG = logging.getLogger()
@@ -87,7 +89,7 @@ def handle_threaded_sql_statements(pool, fd, analyzer):
address = dict(street=row['street'], postcode=row['postcode'])
args = ('SRID=4326;' + row['geometry'],
int(row['from']), int(row['to']), row['interpolation'],
PlaceInfo({'address': address}).analyze(analyzer),
Json(analyzer.process_place(PlaceInfo({'address': address}))),
analyzer.normalize_postcode(row['postcode']))
except ValueError:
continue

12
test/python/cli/test_cmd_import.py
View File

@@ -10,7 +10,7 @@ Tests for import command of the command-line interface wrapper.
import pytest
import nominatim.tools.database_import
import nominatim.tools.country_info
import nominatim.data.country_info
import nominatim.tools.refresh
import nominatim.tools.postcodes
import nominatim.indexer.indexer
@@ -37,7 +37,7 @@ class TestCliImportWithDb:
def test_import_full(self, mock_func_factory, with_updates, place_table, property_table):
mocks = [
mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),
mock_func_factory(nominatim.tools.country_info, 'setup_country_tables'),
mock_func_factory(nominatim.data.country_info, 'setup_country_tables'),
mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
@@ -46,7 +46,7 @@ class TestCliImportWithDb:
mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
mock_func_factory(nominatim.data.country_info, 'create_country_names'),
mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_config'),
mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
@@ -76,7 +76,7 @@ class TestCliImportWithDb:
mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim.tools.database_import, 'load_data'),
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
mock_func_factory(nominatim.data.country_info, 'create_country_names'),
mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim.tools.refresh, 'setup_website'),
@@ -94,7 +94,7 @@ class TestCliImportWithDb:
temp_db_conn):
mocks = [
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
mock_func_factory(nominatim.data.country_info, 'create_country_names'),
mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim.tools.refresh, 'setup_website'),
mock_func_factory(nominatim.db.properties, 'set_property')
@@ -115,7 +115,7 @@ class TestCliImportWithDb:
def test_import_continue_postprocess(self, mock_func_factory):
mocks = [
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
mock_func_factory(nominatim.data.country_info, 'create_country_names'),
mock_func_factory(nominatim.tools.refresh, 'setup_website'),
mock_func_factory(nominatim.db.properties, 'set_property')
]

2
test/python/tools/test_country_info.py → test/python/data/test_country_info.py
View File

@@ -10,7 +10,7 @@ Tests for function that handle country properties.
from textwrap import dedent
import pytest
from nominatim.tools import country_info
from nominatim.data import country_info
@pytest.fixture
def loaded_country(def_config):

2
test/python/dummy_tokenizer.py
View File

@@ -7,7 +7,7 @@
"""
Tokenizer for testing.
"""
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
from nominatim.config import Configuration
def create(dsn, data_dir):

2
test/python/tokenizer/sanitizers/test_clean_housenumbers.py
View File

@@ -10,7 +10,7 @@ Tests for the sanitizer that normalizes housenumbers.
import pytest
from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
@pytest.fixture
def sanitize(request):

4
test/python/tokenizer/sanitizers/test_clean_postcodes.py
View File

@@ -10,8 +10,8 @@ Tests for the sanitizer that normalizes postcodes.
import pytest
from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
from nominatim.indexer.place_info import PlaceInfo
from nominatim.tools import country_info
from nominatim.data.place_info import PlaceInfo
from nominatim.data import country_info
@pytest.fixture
def sanitize(def_config, request):

2
test/python/tokenizer/sanitizers/test_split_name_list.py
View File

@@ -10,7 +10,7 @@ Tests for the sanitizer that splits multivalue lists.
import pytest
from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
from nominatim.errors import UsageError

2
test/python/tokenizer/sanitizers/test_strip_brace_terms.py
View File

@@ -10,7 +10,7 @@ Tests for the sanitizer that handles braced suffixes.
import pytest
from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
def run_sanitizer_on(**kwargs):
place = PlaceInfo({'name': kwargs})

4
test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py
View File

@@ -9,9 +9,9 @@ Tests for the sanitizer that enables language-dependent analyzers.
"""
import pytest
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
from nominatim.tools.country_info import setup_country_config
from nominatim.data.country_info import setup_country_config
class TestWithDefaults:

2
test/python/tokenizer/test_icu.py
View File

@@ -17,7 +17,7 @@ from nominatim.tokenizer import icu_tokenizer
import nominatim.tokenizer.icu_rule_loader
from nominatim.db import properties
from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
from mock_icu_word_table import MockIcuWordTable

2
test/python/tokenizer/test_legacy.py
View File

@@ -12,7 +12,7 @@ import re
import pytest
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
from nominatim.tokenizer import legacy_tokenizer
from nominatim.db import properties
from nominatim.errors import UsageError

2
test/python/tokenizer/test_place_sanitizer.py
View File

@@ -11,7 +11,7 @@ import pytest
from nominatim.errors import UsageError
import nominatim.tokenizer.place_sanitizer as sanitizer
from nominatim.indexer.place_info import PlaceInfo
from nominatim.data.place_info import PlaceInfo
def test_placeinfo_clone_new_name():

3
test/python/tools/test_postcodes.py
View File

@@ -11,7 +11,8 @@ import subprocess
import pytest
from nominatim.tools import postcodes, country_info
from nominatim.tools import postcodes
from nominatim.data import country_info
import dummy_tokenizer
class MockPostcodeTable: