mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-02-26 11:08:13 +00:00
Merge pull request #2393 from lonvia/fix-flake8-issues
Fix flake8 issues
This commit is contained in:
@@ -103,7 +103,7 @@ class CommandlineParser:
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
##### Subcommand classes
|
# Subcommand classes
|
||||||
#
|
#
|
||||||
# Each class needs to implement two functions: add_args() adds the CLI parameters
|
# Each class needs to implement two functions: add_args() adds the CLI parameters
|
||||||
# for the subfunction, run() executes the subcommand.
|
# for the subfunction, run() executes the subcommand.
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ class UpdateRefresh:
|
|||||||
args.threads or 1)
|
args.threads or 1)
|
||||||
indexer.index_postcodes()
|
indexer.index_postcodes()
|
||||||
else:
|
else:
|
||||||
LOG.error("The place table doesn\'t exist. " \
|
LOG.error("The place table doesn't exist. "
|
||||||
"Postcode updates on a frozen database is not possible.")
|
"Postcode updates on a frozen database is not possible.")
|
||||||
|
|
||||||
if args.word_counts:
|
if args.word_counts:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import os
|
|||||||
import psycopg2
|
import psycopg2
|
||||||
import psycopg2.extensions
|
import psycopg2.extensions
|
||||||
import psycopg2.extras
|
import psycopg2.extras
|
||||||
|
from psycopg2 import sql as pysql
|
||||||
|
|
||||||
from nominatim.errors import UsageError
|
from nominatim.errors import UsageError
|
||||||
|
|
||||||
@@ -25,6 +26,16 @@ class _Cursor(psycopg2.extras.DictCursor):
|
|||||||
|
|
||||||
super().execute(query, args)
|
super().execute(query, args)
|
||||||
|
|
||||||
|
|
||||||
|
def execute_values(self, sql, argslist, template=None):
|
||||||
|
""" Wrapper for the psycopg2 convenience function to execute
|
||||||
|
SQL for a list of values.
|
||||||
|
"""
|
||||||
|
LOG.debug("SQL execute_values(%s, %s)", sql, argslist)
|
||||||
|
|
||||||
|
psycopg2.extras.execute_values(self, sql, argslist, template=template)
|
||||||
|
|
||||||
|
|
||||||
def scalar(self, sql, args=None):
|
def scalar(self, sql, args=None):
|
||||||
""" Execute query that returns a single value. The value is returned.
|
""" Execute query that returns a single value. The value is returned.
|
||||||
If the query yields more than one row, a ValueError is raised.
|
If the query yields more than one row, a ValueError is raised.
|
||||||
@@ -37,6 +48,22 @@ class _Cursor(psycopg2.extras.DictCursor):
|
|||||||
return self.fetchone()[0]
|
return self.fetchone()[0]
|
||||||
|
|
||||||
|
|
||||||
|
def drop_table(self, name, if_exists=True, cascade=False):
|
||||||
|
""" Drop the table with the given name.
|
||||||
|
Set `if_exists` to False if a non-existant table should raise
|
||||||
|
an exception instead of just being ignored. If 'cascade' is set
|
||||||
|
to True then all dependent tables are deleted as well.
|
||||||
|
"""
|
||||||
|
sql = 'DROP TABLE '
|
||||||
|
if if_exists:
|
||||||
|
sql += 'IF EXISTS '
|
||||||
|
sql += '{}'
|
||||||
|
if cascade:
|
||||||
|
sql += ' CASCADE'
|
||||||
|
|
||||||
|
self.execute(pysql.SQL(sql).format(pysql.Identifier(name)))
|
||||||
|
|
||||||
|
|
||||||
class _Connection(psycopg2.extensions.connection):
|
class _Connection(psycopg2.extensions.connection):
|
||||||
""" A connection that provides the specialised cursor by default and
|
""" A connection that provides the specialised cursor by default and
|
||||||
adds convenience functions for administrating the database.
|
adds convenience functions for administrating the database.
|
||||||
@@ -75,14 +102,13 @@ class _Connection(psycopg2.extensions.connection):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def drop_table(self, name, if_exists=True):
|
def drop_table(self, name, if_exists=True, cascade=False):
|
||||||
""" Drop the table with the given name.
|
""" Drop the table with the given name.
|
||||||
Set `if_exists` to False if a non-existant table should raise
|
Set `if_exists` to False if a non-existant table should raise
|
||||||
an exception instead of just being ignored.
|
an exception instead of just being ignored.
|
||||||
"""
|
"""
|
||||||
with self.cursor() as cur:
|
with self.cursor() as cur:
|
||||||
cur.execute("""DROP TABLE {} "{}"
|
cur.drop_table(name, if_exists, cascade)
|
||||||
""".format('IF EXISTS' if if_exists else '', name))
|
|
||||||
self.commit()
|
self.commit()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,13 +5,17 @@ tasks.
|
|||||||
import functools
|
import functools
|
||||||
|
|
||||||
import psycopg2.extras
|
import psycopg2.extras
|
||||||
|
from psycopg2 import sql as pysql
|
||||||
|
|
||||||
# pylint: disable=C0111
|
# pylint: disable=C0111
|
||||||
|
|
||||||
|
def _mk_valuelist(template, num):
|
||||||
|
return pysql.SQL(',').join([pysql.SQL(template)] * num)
|
||||||
|
|
||||||
class AbstractPlacexRunner:
|
class AbstractPlacexRunner:
|
||||||
""" Returns SQL commands for indexing of the placex table.
|
""" Returns SQL commands for indexing of the placex table.
|
||||||
"""
|
"""
|
||||||
SELECT_SQL = 'SELECT place_id FROM placex'
|
SELECT_SQL = pysql.SQL('SELECT place_id FROM placex ')
|
||||||
|
|
||||||
def __init__(self, rank, analyzer):
|
def __init__(self, rank, analyzer):
|
||||||
self.rank = rank
|
self.rank = rank
|
||||||
@@ -21,11 +25,12 @@ class AbstractPlacexRunner:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
@functools.lru_cache(maxsize=1)
|
@functools.lru_cache(maxsize=1)
|
||||||
def _index_sql(num_places):
|
def _index_sql(num_places):
|
||||||
return """ UPDATE placex
|
return pysql.SQL(
|
||||||
|
""" UPDATE placex
|
||||||
SET indexed_status = 0, address = v.addr, token_info = v.ti
|
SET indexed_status = 0, address = v.addr, token_info = v.ti
|
||||||
FROM (VALUES {}) as v(id, addr, ti)
|
FROM (VALUES {}) as v(id, addr, ti)
|
||||||
WHERE place_id = v.id
|
WHERE place_id = v.id
|
||||||
""".format(','.join(["(%s, %s::hstore, %s::jsonb)"] * num_places))
|
""").format(_mk_valuelist("(%s, %s::hstore, %s::jsonb)", num_places))
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -52,14 +57,15 @@ class RankRunner(AbstractPlacexRunner):
|
|||||||
return "rank {}".format(self.rank)
|
return "rank {}".format(self.rank)
|
||||||
|
|
||||||
def sql_count_objects(self):
|
def sql_count_objects(self):
|
||||||
return """SELECT count(*) FROM placex
|
return pysql.SQL("""SELECT count(*) FROM placex
|
||||||
WHERE rank_address = {} and indexed_status > 0
|
WHERE rank_address = {} and indexed_status > 0
|
||||||
""".format(self.rank)
|
""").format(pysql.Literal(self.rank))
|
||||||
|
|
||||||
def sql_get_objects(self):
|
def sql_get_objects(self):
|
||||||
return """{} WHERE indexed_status > 0 and rank_address = {}
|
return self.SELECT_SQL + pysql.SQL(
|
||||||
|
"""WHERE indexed_status > 0 and rank_address = {}
|
||||||
ORDER BY geometry_sector
|
ORDER BY geometry_sector
|
||||||
""".format(self.SELECT_SQL, self.rank)
|
""").format(pysql.Literal(self.rank))
|
||||||
|
|
||||||
|
|
||||||
class BoundaryRunner(AbstractPlacexRunner):
|
class BoundaryRunner(AbstractPlacexRunner):
|
||||||
@@ -71,17 +77,18 @@ class BoundaryRunner(AbstractPlacexRunner):
|
|||||||
return "boundaries rank {}".format(self.rank)
|
return "boundaries rank {}".format(self.rank)
|
||||||
|
|
||||||
def sql_count_objects(self):
|
def sql_count_objects(self):
|
||||||
return """SELECT count(*) FROM placex
|
return pysql.SQL("""SELECT count(*) FROM placex
|
||||||
WHERE indexed_status > 0
|
WHERE indexed_status > 0
|
||||||
AND rank_search = {}
|
AND rank_search = {}
|
||||||
AND class = 'boundary' and type = 'administrative'
|
AND class = 'boundary' and type = 'administrative'
|
||||||
""".format(self.rank)
|
""").format(pysql.Literal(self.rank))
|
||||||
|
|
||||||
def sql_get_objects(self):
|
def sql_get_objects(self):
|
||||||
return """{} WHERE indexed_status > 0 and rank_search = {}
|
return self.SELECT_SQL + pysql.SQL(
|
||||||
|
"""WHERE indexed_status > 0 and rank_search = {}
|
||||||
and class = 'boundary' and type = 'administrative'
|
and class = 'boundary' and type = 'administrative'
|
||||||
ORDER BY partition, admin_level
|
ORDER BY partition, admin_level
|
||||||
""".format(self.SELECT_SQL, self.rank)
|
""").format(pysql.Literal(self.rank))
|
||||||
|
|
||||||
|
|
||||||
class InterpolationRunner:
|
class InterpolationRunner:
|
||||||
@@ -120,11 +127,11 @@ class InterpolationRunner:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
@functools.lru_cache(maxsize=1)
|
@functools.lru_cache(maxsize=1)
|
||||||
def _index_sql(num_places):
|
def _index_sql(num_places):
|
||||||
return """ UPDATE location_property_osmline
|
return pysql.SQL("""UPDATE location_property_osmline
|
||||||
SET indexed_status = 0, address = v.addr, token_info = v.ti
|
SET indexed_status = 0, address = v.addr, token_info = v.ti
|
||||||
FROM (VALUES {}) as v(id, addr, ti)
|
FROM (VALUES {}) as v(id, addr, ti)
|
||||||
WHERE place_id = v.id
|
WHERE place_id = v.id
|
||||||
""".format(','.join(["(%s, %s::hstore, %s::jsonb)"] * num_places))
|
""").format(_mk_valuelist("(%s, %s::hstore, %s::jsonb)", num_places))
|
||||||
|
|
||||||
|
|
||||||
def index_places(self, worker, places):
|
def index_places(self, worker, places):
|
||||||
@@ -157,6 +164,6 @@ class PostcodeRunner:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def index_places(worker, ids):
|
def index_places(worker, ids):
|
||||||
worker.perform(""" UPDATE location_postcode SET indexed_status = 0
|
worker.perform(pysql.SQL("""UPDATE location_postcode SET indexed_status = 0
|
||||||
WHERE place_id IN ({})
|
WHERE place_id IN ({})""")
|
||||||
""".format(','.join((str(i[0]) for i in ids))))
|
.format(pysql.SQL(',').join((pysql.Literal(i[0]) for i in ids))))
|
||||||
|
|||||||
@@ -9,8 +9,6 @@ import re
|
|||||||
from textwrap import dedent
|
from textwrap import dedent
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import psycopg2.extras
|
|
||||||
|
|
||||||
from nominatim.db.connection import connect
|
from nominatim.db.connection import connect
|
||||||
from nominatim.db.properties import set_property, get_property
|
from nominatim.db.properties import set_property, get_property
|
||||||
from nominatim.db.utils import CopyBuffer
|
from nominatim.db.utils import CopyBuffer
|
||||||
@@ -359,8 +357,7 @@ class LegacyICUNameAnalyzer:
|
|||||||
to_delete = existing_phrases - new_phrases
|
to_delete = existing_phrases - new_phrases
|
||||||
|
|
||||||
if to_delete:
|
if to_delete:
|
||||||
psycopg2.extras.execute_values(
|
cursor.execute_values(
|
||||||
cursor,
|
|
||||||
""" DELETE FROM word USING (VALUES %s) as v(name, in_class, in_type, op)
|
""" DELETE FROM word USING (VALUES %s) as v(name, in_class, in_type, op)
|
||||||
WHERE word = name and class = in_class and type = in_type
|
WHERE word = name and class = in_class and type = in_type
|
||||||
and ((op = '-' and operator is null) or op = operator)""",
|
and ((op = '-' and operator is null) or op = operator)""",
|
||||||
@@ -386,9 +383,9 @@ class LegacyICUNameAnalyzer:
|
|||||||
if word_tokens:
|
if word_tokens:
|
||||||
cur.execute("""INSERT INTO word (word_id, word_token, country_code,
|
cur.execute("""INSERT INTO word (word_id, word_token, country_code,
|
||||||
search_name_count)
|
search_name_count)
|
||||||
(SELECT nextval('seq_word'), token, '{}', 0
|
(SELECT nextval('seq_word'), token, %s, 0
|
||||||
FROM unnest(%s) as token)
|
FROM unnest(%s) as token)
|
||||||
""".format(country_code), (list(word_tokens),))
|
""", (country_code, list(word_tokens)))
|
||||||
|
|
||||||
|
|
||||||
def process_place(self, place):
|
def process_place(self, place):
|
||||||
|
|||||||
@@ -370,8 +370,7 @@ class LegacyNameAnalyzer:
|
|||||||
to_delete = existing_phrases - norm_phrases
|
to_delete = existing_phrases - norm_phrases
|
||||||
|
|
||||||
if to_add:
|
if to_add:
|
||||||
psycopg2.extras.execute_values(
|
cur.execute_values(
|
||||||
cur,
|
|
||||||
""" INSERT INTO word (word_id, word_token, word, class, type,
|
""" INSERT INTO word (word_id, word_token, word, class, type,
|
||||||
search_name_count, operator)
|
search_name_count, operator)
|
||||||
(SELECT nextval('seq_word'), ' ' || make_standard_name(name), name,
|
(SELECT nextval('seq_word'), ' ' || make_standard_name(name), name,
|
||||||
@@ -381,8 +380,7 @@ class LegacyNameAnalyzer:
|
|||||||
to_add)
|
to_add)
|
||||||
|
|
||||||
if to_delete and should_replace:
|
if to_delete and should_replace:
|
||||||
psycopg2.extras.execute_values(
|
cur.execute_values(
|
||||||
cur,
|
|
||||||
""" DELETE FROM word USING (VALUES %s) as v(name, in_class, in_type, op)
|
""" DELETE FROM word USING (VALUES %s) as v(name, in_class, in_type, op)
|
||||||
WHERE word = name and class = in_class and type = in_type
|
WHERE word = name and class = in_class and type = in_type
|
||||||
and ((op = '-' and operator is null) or op = operator)""",
|
and ((op = '-' and operator is null) or op = operator)""",
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ def _check(hint=None):
|
|||||||
"""
|
"""
|
||||||
def decorator(func):
|
def decorator(func):
|
||||||
title = func.__doc__.split('\n', 1)[0].strip()
|
title = func.__doc__.split('\n', 1)[0].strip()
|
||||||
|
|
||||||
def run_check(conn, config):
|
def run_check(conn, config):
|
||||||
print(title, end=' ... ')
|
print(title, end=' ... ')
|
||||||
ret = func(conn, config)
|
ret = func(conn, config)
|
||||||
@@ -98,13 +99,12 @@ def _get_indexes(conn):
|
|||||||
if conn.table_exists('place'):
|
if conn.table_exists('place'):
|
||||||
indexes.extend(('idx_placex_pendingsector',
|
indexes.extend(('idx_placex_pendingsector',
|
||||||
'idx_location_area_country_place_id',
|
'idx_location_area_country_place_id',
|
||||||
'idx_place_osm_unique'
|
'idx_place_osm_unique'))
|
||||||
))
|
|
||||||
|
|
||||||
return indexes
|
return indexes
|
||||||
|
|
||||||
|
|
||||||
### CHECK FUNCTIONS
|
# CHECK FUNCTIONS
|
||||||
#
|
#
|
||||||
# Functions are exectured in the order they appear here.
|
# Functions are exectured in the order they appear here.
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
import psutil
|
import psutil
|
||||||
import psycopg2.extras
|
import psycopg2.extras
|
||||||
|
from psycopg2 import sql as pysql
|
||||||
|
|
||||||
from nominatim.db.connection import connect, get_pg_env
|
from nominatim.db.connection import connect, get_pg_env
|
||||||
from nominatim.db import utils as db_utils
|
from nominatim.db import utils as db_utils
|
||||||
@@ -184,7 +185,12 @@ def truncate_data_tables(conn):
|
|||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
_COPY_COLUMNS = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'
|
|
||||||
|
_COPY_COLUMNS = pysql.SQL(',').join(map(pysql.Identifier,
|
||||||
|
('osm_type', 'osm_id', 'class', 'type',
|
||||||
|
'name', 'admin_level', 'address',
|
||||||
|
'extratags', 'geometry')))
|
||||||
|
|
||||||
|
|
||||||
def load_data(dsn, threads):
|
def load_data(dsn, threads):
|
||||||
""" Copy data into the word and placex table.
|
""" Copy data into the word and placex table.
|
||||||
@@ -195,12 +201,15 @@ def load_data(dsn, threads):
|
|||||||
for imod in range(place_threads):
|
for imod in range(place_threads):
|
||||||
conn = DBConnection(dsn)
|
conn = DBConnection(dsn)
|
||||||
conn.connect()
|
conn.connect()
|
||||||
conn.perform("""INSERT INTO placex ({0})
|
conn.perform(
|
||||||
SELECT {0} FROM place
|
pysql.SQL("""INSERT INTO placex ({columns})
|
||||||
WHERE osm_id % {1} = {2}
|
SELECT {columns} FROM place
|
||||||
|
WHERE osm_id % {total} = {mod}
|
||||||
AND NOT (class='place' and (type='houses' or type='postcode'))
|
AND NOT (class='place' and (type='houses' or type='postcode'))
|
||||||
AND ST_IsValid(geometry)
|
AND ST_IsValid(geometry)
|
||||||
""".format(_COPY_COLUMNS, place_threads, imod))
|
""").format(columns=_COPY_COLUMNS,
|
||||||
|
total=pysql.Literal(place_threads),
|
||||||
|
mod=pysql.Literal(imod)))
|
||||||
sel.register(conn, selectors.EVENT_READ, conn)
|
sel.register(conn, selectors.EVENT_READ, conn)
|
||||||
|
|
||||||
# Address interpolations go into another table.
|
# Address interpolations go into another table.
|
||||||
@@ -250,6 +259,7 @@ def create_search_indices(conn, config, drop=False):
|
|||||||
|
|
||||||
sql.run_sql_file(conn, 'indices.sql', drop=drop)
|
sql.run_sql_file(conn, 'indices.sql', drop=drop)
|
||||||
|
|
||||||
|
|
||||||
def create_country_names(conn, tokenizer, languages=None):
|
def create_country_names(conn, tokenizer, languages=None):
|
||||||
""" Add default country names to search index. `languages` is a comma-
|
""" Add default country names to search index. `languages` is a comma-
|
||||||
separated list of language codes as used in OSM. If `languages` is not
|
separated list of language codes as used in OSM. If `languages` is not
|
||||||
@@ -261,8 +271,7 @@ def create_country_names(conn, tokenizer, languages=None):
|
|||||||
|
|
||||||
def _include_key(key):
|
def _include_key(key):
|
||||||
return key == 'name' or \
|
return key == 'name' or \
|
||||||
(key.startswith('name:') \
|
(key.startswith('name:') and (not languages or key[5:] in languages))
|
||||||
and (not languages or key[5:] in languages))
|
|
||||||
|
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
psycopg2.extras.register_hstore(cur)
|
psycopg2.extras.register_hstore(cur)
|
||||||
|
|||||||
@@ -141,6 +141,6 @@ def get_url(url):
|
|||||||
try:
|
try:
|
||||||
with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response:
|
with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response:
|
||||||
return response.read().decode('utf-8')
|
return response.read().decode('utf-8')
|
||||||
except:
|
except Exception:
|
||||||
LOG.fatal('Failed to load URL: %s', url)
|
LOG.fatal('Failed to load URL: %s', url)
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ Functions for removing unnecessary data from the database.
|
|||||||
"""
|
"""
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from psycopg2 import sql as pysql
|
||||||
|
|
||||||
UPDATE_TABLES = [
|
UPDATE_TABLES = [
|
||||||
'address_levels',
|
'address_levels',
|
||||||
'gb_postcode',
|
'gb_postcode',
|
||||||
@@ -21,15 +23,15 @@ def drop_update_tables(conn):
|
|||||||
""" Drop all tables only necessary for updating the database from
|
""" Drop all tables only necessary for updating the database from
|
||||||
OSM replication data.
|
OSM replication data.
|
||||||
"""
|
"""
|
||||||
|
parts = (pysql.SQL("(tablename LIKE {})").format(pysql.Literal(t)) for t in UPDATE_TABLES)
|
||||||
where = ' or '.join(["(tablename LIKE '{}')".format(t) for t in UPDATE_TABLES])
|
|
||||||
|
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute("SELECT tablename FROM pg_tables WHERE " + where)
|
cur.execute(pysql.SQL("SELECT tablename FROM pg_tables WHERE ")
|
||||||
|
+ pysql.SQL(' or ').join(parts))
|
||||||
tables = [r[0] for r in cur]
|
tables = [r[0] for r in cur]
|
||||||
|
|
||||||
for table in tables:
|
for table in tables:
|
||||||
cur.execute('DROP TABLE IF EXISTS "{}" CASCADE'.format(table))
|
cur.drop_table(table, cascade=True)
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
|
|||||||
@@ -142,7 +142,8 @@ def change_housenumber_transliteration(conn, **_):
|
|||||||
BEGIN
|
BEGIN
|
||||||
SELECT array_to_string(array_agg(trans), ';')
|
SELECT array_to_string(array_agg(trans), ';')
|
||||||
INTO normtext
|
INTO normtext
|
||||||
FROM (SELECT lookup_word as trans, getorcreate_housenumber_id(lookup_word)
|
FROM (SELECT lookup_word as trans,
|
||||||
|
getorcreate_housenumber_id(lookup_word)
|
||||||
FROM (SELECT make_standard_name(h) as lookup_word
|
FROM (SELECT make_standard_name(h) as lookup_word
|
||||||
FROM regexp_split_to_table(housenumber, '[,;]') h) x) y;
|
FROM regexp_split_to_table(housenumber, '[,;]') h) x) y;
|
||||||
return normtext;
|
return normtext;
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import gzip
|
|||||||
import logging
|
import logging
|
||||||
from math import isfinite
|
from math import isfinite
|
||||||
|
|
||||||
from psycopg2.extras import execute_values
|
from psycopg2 import sql as pysql
|
||||||
|
|
||||||
from nominatim.db.connection import connect
|
from nominatim.db.connection import connect
|
||||||
|
|
||||||
@@ -52,27 +52,26 @@ class _CountryPostcodesCollector:
|
|||||||
|
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
if to_add:
|
if to_add:
|
||||||
execute_values(cur,
|
cur.execute_values(
|
||||||
"""INSERT INTO location_postcode
|
"""INSERT INTO location_postcode
|
||||||
(place_id, indexed_status, country_code,
|
(place_id, indexed_status, country_code,
|
||||||
postcode, geometry) VALUES %s""",
|
postcode, geometry) VALUES %s""",
|
||||||
to_add,
|
to_add,
|
||||||
template="""(nextval('seq_place'), 1, '{}',
|
template=pysql.SQL("""(nextval('seq_place'), 1, {},
|
||||||
%s, 'SRID=4326;POINT(%s %s)')
|
%s, 'SRID=4326;POINT(%s %s)')
|
||||||
""".format(self.country))
|
""").format(pysql.Literal(self.country)))
|
||||||
if to_delete:
|
if to_delete:
|
||||||
cur.execute("""DELETE FROM location_postcode
|
cur.execute("""DELETE FROM location_postcode
|
||||||
WHERE country_code = %s and postcode = any(%s)
|
WHERE country_code = %s and postcode = any(%s)
|
||||||
""", (self.country, to_delete))
|
""", (self.country, to_delete))
|
||||||
if to_update:
|
if to_update:
|
||||||
execute_values(cur,
|
cur.execute_values(
|
||||||
"""UPDATE location_postcode
|
pysql.SQL("""UPDATE location_postcode
|
||||||
SET indexed_status = 2,
|
SET indexed_status = 2,
|
||||||
geometry = ST_SetSRID(ST_Point(v.x, v.y), 4326)
|
geometry = ST_SetSRID(ST_Point(v.x, v.y), 4326)
|
||||||
FROM (VALUES %s) AS v (pc, x, y)
|
FROM (VALUES %s) AS v (pc, x, y)
|
||||||
WHERE country_code = '{}' and postcode = pc
|
WHERE country_code = {} and postcode = pc
|
||||||
""".format(self.country),
|
""").format(pysql.Literal(self.country)), to_update)
|
||||||
to_update)
|
|
||||||
|
|
||||||
|
|
||||||
def _compute_changes(self, conn):
|
def _compute_changes(self, conn):
|
||||||
@@ -166,10 +165,13 @@ def update_postcodes(dsn, project_dir, tokenizer):
|
|||||||
cur.execute("""
|
cur.execute("""
|
||||||
SELECT cc as country_code, pc, ST_X(centroid), ST_Y(centroid)
|
SELECT cc as country_code, pc, ST_X(centroid), ST_Y(centroid)
|
||||||
FROM (SELECT
|
FROM (SELECT
|
||||||
COALESCE(plx.country_code, get_country_code(ST_Centroid(pl.geometry))) as cc,
|
COALESCE(plx.country_code,
|
||||||
|
get_country_code(ST_Centroid(pl.geometry))) as cc,
|
||||||
token_normalized_postcode(pl.address->'postcode') as pc,
|
token_normalized_postcode(pl.address->'postcode') as pc,
|
||||||
ST_Centroid(ST_Collect(COALESCE(plx.centroid, ST_Centroid(pl.geometry)))) as centroid
|
ST_Centroid(ST_Collect(COALESCE(plx.centroid,
|
||||||
FROM place AS pl LEFT OUTER JOIN placex AS plx ON pl.osm_id = plx.osm_id AND pl.osm_type = plx.osm_type
|
ST_Centroid(pl.geometry)))) as centroid
|
||||||
|
FROM place AS pl LEFT OUTER JOIN placex AS plx
|
||||||
|
ON pl.osm_id = plx.osm_id AND pl.osm_type = plx.osm_type
|
||||||
WHERE pl.address ? 'postcode' AND pl.geometry IS NOT null
|
WHERE pl.address ? 'postcode' AND pl.geometry IS NOT null
|
||||||
GROUP BY cc, pc) xx
|
GROUP BY cc, pc) xx
|
||||||
WHERE pc IS NOT null AND cc IS NOT null
|
WHERE pc IS NOT null AND cc IS NOT null
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
from textwrap import dedent
|
from textwrap import dedent
|
||||||
|
|
||||||
from psycopg2.extras import execute_values
|
from psycopg2 import sql as pysql
|
||||||
|
|
||||||
from nominatim.db.utils import execute_file
|
from nominatim.db.utils import execute_file
|
||||||
from nominatim.db.sql_preprocessor import SQLPreprocessor
|
from nominatim.db.sql_preprocessor import SQLPreprocessor
|
||||||
@@ -49,7 +49,7 @@ def load_address_levels(conn, table, levels):
|
|||||||
_add_address_level_rows_from_entry(rows, entry)
|
_add_address_level_rows_from_entry(rows, entry)
|
||||||
|
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute('DROP TABLE IF EXISTS {}'.format(table))
|
cur.drop_table(table)
|
||||||
|
|
||||||
cur.execute("""CREATE TABLE {} (country_code varchar(2),
|
cur.execute("""CREATE TABLE {} (country_code varchar(2),
|
||||||
class TEXT,
|
class TEXT,
|
||||||
@@ -57,7 +57,8 @@ def load_address_levels(conn, table, levels):
|
|||||||
rank_search SMALLINT,
|
rank_search SMALLINT,
|
||||||
rank_address SMALLINT)""".format(table))
|
rank_address SMALLINT)""".format(table))
|
||||||
|
|
||||||
execute_values(cur, "INSERT INTO {} VALUES %s".format(table), rows)
|
cur.execute_values(pysql.SQL("INSERT INTO {} VALUES %s")
|
||||||
|
.format(pysql.Identifier(table)), rows)
|
||||||
|
|
||||||
cur.execute('CREATE UNIQUE INDEX ON {} (country_code, class, type)'.format(table))
|
cur.execute('CREATE UNIQUE INDEX ON {} (country_code, class, type)'.format(table))
|
||||||
|
|
||||||
|
|||||||
@@ -174,10 +174,10 @@ class SPImporter():
|
|||||||
|
|
||||||
if table_name in self.table_phrases_to_delete:
|
if table_name in self.table_phrases_to_delete:
|
||||||
self.statistics_handler.notify_one_table_ignored()
|
self.statistics_handler.notify_one_table_ignored()
|
||||||
#Remove this table from the ones to delete as it match a class/type
|
# Remove this table from the ones to delete as it match a
|
||||||
#still existing on the special phrases of the wiki.
|
# class/type still existing on the special phrases of the wiki.
|
||||||
self.table_phrases_to_delete.remove(table_name)
|
self.table_phrases_to_delete.remove(table_name)
|
||||||
#So dont need to create the table and indexes.
|
# So don't need to create the table and indexes.
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Table creation
|
# Table creation
|
||||||
@@ -248,18 +248,14 @@ class SPImporter():
|
|||||||
Delete the place_classtype tables.
|
Delete the place_classtype tables.
|
||||||
"""
|
"""
|
||||||
LOG.warning('Cleaning database...')
|
LOG.warning('Cleaning database...')
|
||||||
#Array containing all queries to execute. Contain tuples of format (query, parameters)
|
|
||||||
queries_parameters = []
|
|
||||||
|
|
||||||
#Delete place_classtype tables corresponding to class/type which are not on the wiki anymore
|
# Delete place_classtype tables corresponding to class/type which
|
||||||
|
# are not on the wiki anymore.
|
||||||
|
with self.db_connection.cursor() as db_cursor:
|
||||||
for table in self.table_phrases_to_delete:
|
for table in self.table_phrases_to_delete:
|
||||||
self.statistics_handler.notify_one_table_deleted()
|
self.statistics_handler.notify_one_table_deleted()
|
||||||
query = SQL('DROP TABLE IF EXISTS {}').format(Identifier(table))
|
db_cursor.drop_table(table)
|
||||||
queries_parameters.append((query, ()))
|
|
||||||
|
|
||||||
with self.db_connection.cursor() as db_cursor:
|
|
||||||
for query, parameters in queries_parameters:
|
|
||||||
db_cursor.execute(query, parameters)
|
|
||||||
|
|
||||||
def _convert_php_settings_if_needed(self, file_path):
|
def _convert_php_settings_if_needed(self, file_path):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -65,5 +65,6 @@ class SPWikiLoader(Iterator):
|
|||||||
Requested URL Example :
|
Requested URL Example :
|
||||||
https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/EN
|
https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/EN
|
||||||
"""
|
"""
|
||||||
url = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/' + lang.upper() # pylint: disable=line-too-long
|
url = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/' \
|
||||||
|
+ lang.upper()
|
||||||
return get_url(url)
|
return get_url(url)
|
||||||
|
|||||||
Reference in New Issue
Block a user