hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-09 19:44:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Linting fixes

Browse Source
This commit is contained in:
Darkshredder
2021-03-08 22:59:51 +05:30
parent 2af82975cd
commit 122c4618b9
3 changed files with 26 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
nominatim/cli.py
View File

@@ -168,10 +168,10 @@ class UpdateAddData:
def run(args): def run(args):
if args.tiger_data: if args.tiger_data:
return tiger_data.add_tiger_data(args.config.get_libpq_dsn(), return tiger_data.add_tiger_data(args.config.get_libpq_dsn(),
args.tiger_data, args.tiger_data,
args.threads or 1, args.threads or 1,
args.config, args.config,
args.sqllib_dir) args.sqllib_dir)
params = ['update.php'] params = ['update.php']
if args.file: if args.file:

2
nominatim/clicmd/transition.py
View File

@@ -129,7 +129,7 @@ class AdminTransition:
LOG.warning('Create Search indices') LOG.warning('Create Search indices')
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop) database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop)
if args.tiger_data: if args.tiger_data:
LOG.warning('Tiger data') LOG.warning('Tiger data')
tiger_data.add_tiger_data(args.config.get_libpq_dsn(), tiger_data.add_tiger_data(args.config.get_libpq_dsn(),

40
nominatim/tools/tiger_data.py
View File

@@ -3,7 +3,6 @@ Functions for setting up and importing a new Nominatim database.
""" """
import logging import logging
import os import os
import time
import tarfile import tarfile
import selectors import selectors
@@ -11,6 +10,9 @@ from ..db.connection import connect
from ..db.async_connection import DBConnection from ..db.async_connection import DBConnection
from ..db.sql_preprocessor import SQLPreprocessor from ..db.sql_preprocessor import SQLPreprocessor
# pylint: disable=R0912
# pylint: disable=R0914,R0915,W0702
LOG = logging.getLogger() LOG = logging.getLogger()
@@ -19,22 +21,22 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
""" """
# Handling directory or tarball file. # Handling directory or tarball file.
is_tarfile = False is_tarfile = False
if(data_dir.endswith('.tar.gz')): if data_dir.endswith('.tar.gz'):
is_tarfile = True is_tarfile = True
tar = tarfile.open(data_dir) tar = tarfile.open(data_dir)
sql_files = [i for i in tar.getmembers() if i.name.endswith('.sql')] sql_files = [i for i in tar.getmembers() if i.name.endswith('.sql')]
LOG.warning(f'Found {len(sql_files)} SQL files in tarfile with path {data_dir}') LOG.warning("Found %d SQL files in tarfile with path %s", len(sql_files), data_dir)
if(not len(sql_files)): if not sql_files:
LOG.warning(f'Tiger data import selected but no files found in tarfile with path {data_dir}') LOG.warning("Tiger data import selected but no files in tarfile's path %s", data_dir)
return return
else: else:
files = os.listdir(data_dir) files = os.listdir(data_dir)
sql_files = [i for i in files if i.endswith('.sql')] sql_files = [i for i in files if i.endswith('.sql')]
LOG.warning(f'Found {len(sql_files)} SQL files in path {data_dir}') LOG.warning("Found %d SQL files in path %s", len(sql_files), data_dir)
if(not len(sql_files)): if not sql_files:
LOG.warning(f'Tiger data import selected but no files found in path {data_dir}') LOG.warning("Tiger data import selected but no files found in path %s", data_dir)
return return
with connect(dsn) as conn: with connect(dsn) as conn:
sql = SQLPreprocessor(conn, config, sqllib_dir) sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'tiger_import_start.sql') sql.run_sql_file(conn, 'tiger_import_start.sql')
@@ -44,7 +46,7 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
sel = selectors.DefaultSelector() sel = selectors.DefaultSelector()
place_threads = max(1, threads - 1) place_threads = max(1, threads - 1)
for sql_file in sql_files: for sql_file in sql_files:
if(not is_tarfile): if not is_tarfile:
file_path = os.path.join(data_dir, sql_file) file_path = os.path.join(data_dir, sql_file)
file = open(file_path) file = open(file_path)
else: else:
@@ -52,17 +54,17 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
lines = 0 lines = 0
end_of_file = False end_of_file = False
total_used_threads = place_threads total_used_threads = place_threads
while(True): while True :
if(end_of_file): if end_of_file:
break break
for imod in range(place_threads): for imod in range(place_threads):
conn = DBConnection(dsn) conn = DBConnection(dsn)
conn.connect() conn.connect()
sql_query = file.readline() sql_query = file.readline()
lines+=1 lines += 1
if(not sql_query): if not sql_query:
end_of_file = True end_of_file = True
total_used_threads = imod total_used_threads = imod
break break
@@ -70,11 +72,11 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
conn.perform(sql_query) conn.perform(sql_query)
sel.register(conn, selectors.EVENT_READ, conn) sel.register(conn, selectors.EVENT_READ, conn)
if(lines==1000): if lines == 1000:
print('. ', end='', flush=True) print('. ', end='', flush=True)
lines=0 lines = 0
todo = min(place_threads,total_used_threads) todo = min(place_threads, total_used_threads)
while todo > 0: while todo > 0:
for key, _ in sel.select(1): for key, _ in sel.select(1):
try: try:
@@ -84,9 +86,9 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
conn.close() conn.close()
todo -= 1 todo -= 1
except: except:
todo -=1 todo -= 1
if(is_tarfile): if is_tarfile:
tar.close() tar.close()
print('\n') print('\n')
LOG.warning("Creating indexes on Tiger data") LOG.warning("Creating indexes on Tiger data")