hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

move update code for postcode and word count to Python

Browse Source 
Adds also tests for the new function to execute a SQL script.
This commit is contained in:
Sarah Hoffmann
2021-01-22 23:25:37 +01:00
parent e6d9485c4a
commit e6c2842b66
6 changed files with 114 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
nominatim/cli.py
View File

@@ -8,9 +8,13 @@ import argparse
import logging
from pathlib import Path
import psycopg2
from .config import Configuration
from .tools.exec_utils import run_legacy_script, run_api_script
LOG = logging.getLogger()
def _num_system_cpus():
try:
cpus = len(os.sched_getaffinity(0))
@@ -366,32 +370,35 @@ class UpdateRefresh:
@staticmethod
def run(args):
if args.postcodes:
run_legacy_script('update.php', '--calculate-postcodes',
nominatim_env=args, throw_on_fail=True)
if args.word_counts:
run_legacy_script('update.php', '--recompute-word-counts',
nominatim_env=args, throw_on_fail=True)
if args.address_levels:
run_legacy_script('update.php', '--update-address-levels',
nominatim_env=args, throw_on_fail=True)
if args.functions:
params = ['setup.php', '--create-functions', '--create-partition-functions']
if args.diffs:
params.append('--enable-diff-updates')
if args.enable_debug_statements:
params.append('--enable-debug-statements')
run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
if args.wiki_data:
run_legacy_script('setup.php', '--import-wikipedia-articles',
nominatim_env=args, throw_on_fail=True)
# Attention: importance MUST come after wiki data import.
if args.importance:
run_legacy_script('update.php', '--recompute-importance',
nominatim_env=args, throw_on_fail=True)
if args.website:
run_legacy_script('setup.php', '--setup-website',
nominatim_env=args, throw_on_fail=True)
import nominatim.tools.refresh
with psycopg2.connect(args.config.get_libpq_dsn()) as conn:
if args.postcodes:
LOG.warning("Update postcodes centroid")
nominatim.tools.refresh.update_postcodes(conn, args.data_dir)
if args.word_counts:
LOG.warning('Recompute frequency of full-word search terms')
nominatim.tools.refresh.recompute_word_counts(conn, args.data_dir)
if args.address_levels:
run_legacy_script('update.php', '--update-address-levels',
nominatim_env=args, throw_on_fail=True)
if args.functions:
params = ['setup.php', '--create-functions', '--create-partition-functions']
if args.diffs:
params.append('--enable-diff-updates')
if args.enable_debug_statements:
params.append('--enable-debug-statements')
run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
if args.wiki_data:
run_legacy_script('setup.php', '--import-wikipedia-articles',
nominatim_env=args, throw_on_fail=True)
# Attention: importance MUST come after wiki data import.
if args.importance:
run_legacy_script('update.php', '--recompute-importance',
nominatim_env=args, throw_on_fail=True)
if args.website:
run_legacy_script('setup.php', '--setup-website',
nominatim_env=args, throw_on_fail=True)
return 0

11
nominatim/db/utils.py Normal file
View File

@@ -0,0 +1,11 @@
"""
Helper functions for handling DB accesses.
"""
def execute_file(conn, fname):
""" Read an SQL file and run its contents against the given connection.
"""
with fname.open('r') as fdesc:
sql = fdesc.read()
with conn.cursor() as cur:
cur.execute(sql)

16
nominatim/tools/refresh.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Functions for bringing auxiliary data in the database up-to-date.
"""
from ..db.utils import execute_file
def update_postcodes(conn, datadir):
""" Recalculate postcode centroids and add, remove and update entries in the
location_postcode table. `conn` is an opne connection to the database.
"""
execute_file(conn, datadir / 'sql' / 'update-postcodes.sql')
def recompute_word_counts(conn, datadir):
""" Compute the frequency of full-word search terms.
"""
execute_file(conn, datadir / 'sql' / 'words_from_search_name.sql')