hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-11 13:24:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

simplify interface for adding tiger data

Browse Source 
Also simplifies tests using existing fixtures.
This commit is contained in:
Sarah Hoffmann
2021-04-19 09:23:37 +02:00
parent 995ba2c7c2
commit 8f63f9516b
4 changed files with 31 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
nominatim/cli.py
View File

@@ -148,11 +148,8 @@ class UpdateAddData:
@staticmethod @staticmethod
def run(args): def run(args):
if args.tiger_data: if args.tiger_data:
return tiger_data.add_tiger_data(args.config.get_libpq_dsn(), return tiger_data.add_tiger_data(args.tiger_data,
args.tiger_data, args.config, args.threads or 1)
args.threads or 1,
args.config,
args.sqllib_dir)
params = ['update.php'] params = ['update.php']
if args.file: if args.file:

10
nominatim/tools/tiger_data.py
View File

@@ -76,17 +76,17 @@ def handle_unregister_connection_pool(sel, place_threads):
conn.close() conn.close()
place_threads -= 1 place_threads -= 1
def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): def add_tiger_data(data_dir, config, threads):
""" Import tiger data from directory or tar file """ Import tiger data from directory or tar file `data dir`.
""" """
dsn = config.get_libpq_dsn()
sql_files, tar = handle_tarfile_or_directory(data_dir) sql_files, tar = handle_tarfile_or_directory(data_dir)
if not sql_files: if not sql_files:
return return
with connect(dsn) as conn: with connect(dsn) as conn:
sql = SQLPreprocessor(conn, config, sqllib_dir) sql = SQLPreprocessor(conn, config, config.lib_dir.sql)
sql.run_sql_file(conn, 'tiger_import_start.sql') sql.run_sql_file(conn, 'tiger_import_start.sql')
# Reading sql_files and then for each file line handling # Reading sql_files and then for each file line handling
@@ -116,5 +116,5 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
print('\n') print('\n')
LOG.warning("Creating indexes on Tiger data") LOG.warning("Creating indexes on Tiger data")
with connect(dsn) as conn: with connect(dsn) as conn:
sql = SQLPreprocessor(conn, config, sqllib_dir) sql = SQLPreprocessor(conn, config, config.lib_dir.sql)
sql.run_sql_file(conn, 'tiger_import_finish.sql') sql.run_sql_file(conn, 'tiger_import_finish.sql')

7
test/python/conftest.py
View File

@@ -126,7 +126,12 @@ def table_factory(temp_db_cursor):
@pytest.fixture @pytest.fixture
def def_config(): def def_config():
return Configuration(None, SRC_DIR.resolve() / 'settings') cfg = Configuration(None, SRC_DIR.resolve() / 'settings')
cfg.set_libdirs(module='.', osm2pgsql='.',
php=SRC_DIR / 'lib-php',
sql=SRC_DIR / 'lib-sql',
data=SRC_DIR / 'data')
return cfg
@pytest.fixture @pytest.fixture
def src_dir(): def src_dir():

41
test/python/test_tools_tiger_data.py
View File

@@ -10,35 +10,31 @@ from nominatim.tools import tiger_data, database_import
@pytest.mark.parametrize("threads", (1, 5)) @pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor, def test_add_tiger_data(def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db): temp_db_cursor, threads, temp_db_with_extensions):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)') temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.sql' sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1); sqlfile.write_text("""INSERT INTO place values (1);
INSERT INTO non_existant_table values (1);""") INSERT INTO non_existant_table values (1);""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql') tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 1 assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5)) @pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_bad_file(dsn, src_dir, def_config, tmp_path, sql_preprocessor, def test_add_tiger_data_bad_file(def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db): temp_db_cursor, threads, temp_db_with_extensions):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)') temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.txt' sqlfile = tmp_path / '1010.txt'
sqlfile.write_text("""Random text""") sqlfile.write_text("""Random text""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql') tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 0 assert temp_db_cursor.table_rows('place') == 0
@pytest.mark.parametrize("threads", (1, 5)) @pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path, def test_add_tiger_data_tarfile(def_config, tmp_path, temp_db_cursor,
temp_db_cursor, threads, temp_db, sql_preprocessor): threads, temp_db_with_extensions, sql_preprocessor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)') temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.sql' sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1); sqlfile.write_text("""INSERT INTO place values (1);
@@ -46,21 +42,20 @@ def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
tar = tarfile.open("sample.tar.gz", "w:gz") tar = tarfile.open("sample.tar.gz", "w:gz")
tar.add(sqlfile) tar.add(sqlfile)
tar.close() tar.close()
tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql') tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 1 assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5)) @pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_bad_tarfile(dsn, src_dir, def_config, tmp_path, def test_add_tiger_data_bad_tarfile(def_config, tmp_path, temp_db_cursor, threads,
temp_db_cursor, threads, temp_db, sql_preprocessor): temp_db_with_extensions, sql_preprocessor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)') temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.txt' sqlfile = tmp_path / '1010.txt'
sqlfile.write_text("""Random text""") sqlfile.write_text("""Random text""")
tar = tarfile.open("sample.tar.gz", "w:gz") tar = tarfile.open("sample.tar.gz", "w:gz")
tar.add(sqlfile) tar.add(sqlfile)
tar.close() tar.close()
tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql') tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 0 assert temp_db_cursor.table_rows('place') == 0