hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

adapt osm2pgsql tests for new address column

Browse Source
This commit is contained in:
Sarah Hoffmann
2017-03-25 23:12:51 +01:00
parent c8e79397f5
commit 49b5e4dbad
5 changed files with 77 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
test/bdd/environment.py
View File

@@ -4,6 +4,7 @@ import os
import psycopg2 import psycopg2
import psycopg2.extras import psycopg2.extras
import subprocess import subprocess
import tempfile
from nose.tools import * # for assert functions from nose.tools import * # for assert functions
from sys import version_info as python_version from sys import version_info as python_version
@@ -88,17 +89,20 @@ class NominatimEnvironment(object):
conn.commit() conn.commit()
conn.close() conn.close()
# execute osm2pgsql on an empty file to get the right tables # execute osm2pgsql import on an empty file to get the right tables
osm2pgsql = os.path.join(self.build_dir, 'osm2pgsql', 'osm2pgsql') with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
proc = subprocess.Popen([osm2pgsql, '-lsc', '-r', 'xml', fd.write(b'<osm version="0.6"></osm>')
'-O', 'gazetteer', '-d', self.template_db, '-'], fd.flush()
cwd=self.build_dir, stdin=subprocess.PIPE, self.run_setup_script('import-data',
stdout=subprocess.PIPE, stderr=subprocess.PIPE) 'ignore-errors',
[outstr, errstr] = proc.communicate(input=b'<osm version="0.6"></osm>') 'create-functions',
logger.debug("running osm2pgsql for template: %s\n%s\n%s" % (osm2pgsql, outstr, errstr)) 'create-tables',
self.run_setup_script('create-functions', 'create-tables', 'create-partition-tables',
'create-partition-tables', 'create-partition-functions', 'create-partition-functions',
'load-data', 'create-search-indices') 'load-data',
'create-search-indices',
osm_file=fd.name,
osm2pgsql_cache='200')
def setup_api_db(self, context): def setup_api_db(self, context):
self.write_nominatim_config(self.api_test_db) self.write_nominatim_config(self.api_test_db)

85
test/bdd/osm2pgsql/import/tags.feature
View File

@@ -102,8 +102,8 @@ Feature: Tag evaluation
n1 T<key>=<value>,name=real n1 T<key>=<value>,name=real
""" """
Then place contains Then place contains
| object | name | | object | class | type | name |
| N1 | 'name' : 'real' | | N1 | <key> | <value> | 'name' : 'real' |
Examples: Examples:
| key | value | | key | value |
@@ -300,7 +300,7 @@ Feature: Tag evaluation
n11 T<key>=<value> n11 T<key>=<value>
""" """
Then place contains Then place contains
| object | class | type | postcode | | object | class | type | addr+postcode |
| N10 | highway | secondary | <value> | | N10 | highway | secondary | <value> |
| N11 | place | postcode | <value> | | N11 | place | postcode | <value> |
And place has no entry for N10:place And place has no entry for N10:place
@@ -320,9 +320,9 @@ Feature: Tag evaluation
n20 Tamenity=hospital,addr:place=Foo%20%Town n20 Tamenity=hospital,addr:place=Foo%20%Town
""" """
Then place contains Then place contains
| object | class | type | street | addr_place | | object | class | type | addr+street | addr+place |
| N10 | amenity | hospital | Foo St | None | | N10 | amenity | hospital | Foo St | - |
| N20 | amenity | hospital | - | Foo Town | | N20 | amenity | hospital | - | Foo Town |
Scenario Outline: Import of country Scenario Outline: Import of country
@@ -331,16 +331,16 @@ Feature: Tag evaluation
n10 Tplace=village,<key>=<value> n10 Tplace=village,<key>=<value>
""" """
Then place contains Then place contains
| object | class | type | country_code | | object | class | type | addr+country |
| N10 | place | village | <value> | | N10 | place | village | <value> |
Examples: Examples:
| key | value | | key | value |
| country_code | us | | country_code | us |
| ISO3166-1 | XX | | ISO3166-1 | XX |
| is_in:country_code | __ | | is_in:country_code | __ |
| addr:country | .. | | addr:country | .. |
| addr:country_code | cv | | addr:country_code | cv |
Scenario Outline: Ignore country codes with wrong length Scenario Outline: Ignore country codes with wrong length
When loading osm data When loading osm data
@@ -348,7 +348,7 @@ Feature: Tag evaluation
n10 Tplace=village,country_code=<value> n10 Tplace=village,country_code=<value>
""" """
Then place contains Then place contains
| object | class | type | country_code | | object | class | type | addr+country |
| N10 | place | village | - | | N10 | place | village | - |
Examples: Examples:
@@ -368,24 +368,11 @@ Feature: Tag evaluation
n13 Tbuilding=yes,addr:conscriptionnumber=3,addr:streetnumber=111 n13 Tbuilding=yes,addr:conscriptionnumber=3,addr:streetnumber=111
""" """
Then place contains Then place contains
| object | class | type | housenumber | | object | class | type | address |
| N10 | building | yes | 4b | | N10 | building | yes | 'housenumber' : '4b' |
| N11 | building | yes | 003 | | N11 | building | yes | 'conscriptionnumber' : '003' |
| N12 | building | yes | 2345 | | N12 | building | yes | 'streetnumber' : '2345' |
| N13 | building | yes | 3/111 | | N13 | building | yes | 'conscriptionnumber' : '3', 'streetnumber' : '111' |
Scenario: Import of address interpolations
When loading osm data
"""
n10 Taddr:interpolation=odd
n11 Taddr:housenumber=10,addr:interpolation=odd
n12 Taddr:interpolation=odd,addr:housenumber=23
"""
Then place contains
| object | class | type | housenumber |
| N10 | place | houses | odd |
| N11 | place | houses | odd |
| N12 | place | houses | odd |
Scenario: Shorten tiger:county tags Scenario: Shorten tiger:county tags
When loading osm data When loading osm data
@@ -395,26 +382,26 @@ Feature: Tag evaluation
n12 Tplace=village,tiger:county=Feebourgh n12 Tplace=village,tiger:county=Feebourgh
""" """
Then place contains Then place contains
| object | class | type | isin | | object | class | type | addr+tiger:county |
| N10 | place | village | Feebourgh county | | N10 | place | village | Feebourgh county |
| N11 | place | village | Alabama,Feebourgh county | | N11 | place | village | Feebourgh county |
| N12 | place | village | Feebourgh county | | N12 | place | village | Feebourgh county |
Scenario Outline: Import of address tags Scenario Outline: Import of address tags
When loading osm data When loading osm data
""" """
n10 Tplace=village,<key>=<value> n10 Tplace=village,addr:<key>=<value>
n11 Tplace=village,is_in:<key>=<value>
""" """
Then place contains Then place contains
| object | class | type | isin | | object | class | type | address |
| N10 | place | village | <value> | | N10 | place | village | '<key>' : '<value>' |
Examples: Examples:
| key | value | | key | value |
| is_in:country | Xanadu | | suburb | hinein |
| addr:suburb | hinein | | city | Sydney |
| addr:city | Sydney | | state | Jura |
| addr:state | Jura |
Scenario: Import of isin tags with space Scenario: Import of isin tags with space
When loading osm data When loading osm data
@@ -423,9 +410,9 @@ Feature: Tag evaluation
n11 Tplace=village,addr:county=le%20%havre n11 Tplace=village,addr:county=le%20%havre
""" """
Then place contains Then place contains
| object | class | type | isin | | object | class | type | address |
| N10 | place | village | Stockholm, Sweden | | N10 | place | village | 'is_in' : 'Stockholm, Sweden' |
| N11 | place | village | le havre | | N11 | place | village | 'county' : 'le havre' |
Scenario: Import of admin level Scenario: Import of admin level
When loading osm data When loading osm data
@@ -438,8 +425,8 @@ Feature: Tag evaluation
Then place contains Then place contains
| object | class | type | admin_level | | object | class | type | admin_level |
| N10 | amenity | hospital | 3 | | N10 | amenity | hospital | 3 |
| N11 | amenity | hospital | 100 | | N11 | amenity | hospital | 15 |
| N12 | amenity | hospital | 100 | | N12 | amenity | hospital | 15 |
| N13 | amenity | hospital | 3 | | N13 | amenity | hospital | 3 |
Scenario Outline: Import of extra tags Scenario Outline: Import of extra tags
@@ -550,5 +537,5 @@ Feature: Tag evaluation
n290393920 Taddr:city=Perpignan,addr:country=FR,addr:housenumber=43\,addr:postcode=66000,addr:street=Rue%20%Pierre%20%Constant%20%d`Ivry,source=cadastre-dgi-fr%20%source%20%:%20%Direction%20%Générale%20%des%20%Impôts%20%-%20%Cadastre%20%;%20%mise%20%à%20%jour%20%:2008 n290393920 Taddr:city=Perpignan,addr:country=FR,addr:housenumber=43\,addr:postcode=66000,addr:street=Rue%20%Pierre%20%Constant%20%d`Ivry,source=cadastre-dgi-fr%20%source%20%:%20%Direction%20%Générale%20%des%20%Impôts%20%-%20%Cadastre%20%;%20%mise%20%à%20%jour%20%:2008
""" """
Then place contains Then place contains
| object | class | type | housenumber | | object | class | type | address |
| N290393920 | place | house| 43\ | | N290393920 | place | house| 'city' : 'Perpignan', 'country' : 'FR', 'housenumber' : '43\\', 'postcode' : '66000', 'street' : 'Rue Pierre Constant d`Ivry' |

4
test/bdd/osm2pgsql/update/relation.feature
View File

@@ -137,7 +137,7 @@ Feature: Update of relations by osm2pgsql
r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=XX,admin_level=2 Mw1@ r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=XX,admin_level=2 Mw1@
""" """
Then place contains Then place contains
| object | country_code | name | | object | addr+country | name |
| R1 | XX | 'name' : 'Foo' | | R1 | XX | 'name' : 'Foo' |
Scenario: Country boundary names are extended when country_code known Scenario: Country boundary names are extended when country_code known
@@ -154,6 +154,6 @@ Feature: Update of relations by osm2pgsql
r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=ch,admin_level=2 Mw1@ r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=ch,admin_level=2 Mw1@
""" """
Then place contains Then place contains
| object | country_code | name+name:de | name+name | | object | addr+country | name+name:de | name+name |
| R1 | ch | Schweiz | Foo | | R1 | ch | Schweiz | Foo |

33
test/bdd/steps/db_ops.py
View File

@@ -20,9 +20,10 @@ class PlaceColumn:
self.add_hstore('name', key[5:], value) self.add_hstore('name', key[5:], value)
elif key.startswith('extra+'): elif key.startswith('extra+'):
self.add_hstore('extratags', key[6:], value) self.add_hstore('extratags', key[6:], value)
elif key.startswith('addr+'):
self.add_hstore('address', key[6:], value)
else: else:
assert_in(key, ('class', 'type', 'street', 'addr_place', assert_in(key, ('class', 'type'))
'isin', 'postcode'))
self.columns[key] = None if value == '' else value self.columns[key] = None if value == '' else value
def set_key_name(self, value): def set_key_name(self, value):
@@ -39,10 +40,16 @@ class PlaceColumn:
self.columns['admin_level'] = int(value) self.columns['admin_level'] = int(value)
def set_key_housenr(self, value): def set_key_housenr(self, value):
self.columns['housenumber'] = None if value == '' else value self.add_hstore('address', 'housenumber', None if value == '' else value)
def set_key_street(self, value):
self.add_hstore('address', 'street', None if value == '' else value)
def set_key_addr_place(self, value):
self.add_hstore('address', 'place', None if value == '' else value)
def set_key_country(self, value): def set_key_country(self, value):
self.columns['country_code'] = None if value == '' else value self.add_hstore('address', 'country', None if value == '' else value)
def set_key_geometry(self, value): def set_key_geometry(self, value):
self.geometry = self.context.osm.parse_geometry(value, self.context.scene) self.geometry = self.context.osm.parse_geometry(value, self.context.scene)
@@ -223,15 +230,11 @@ def import_and_index_data_from_place_table(context):
cur = context.db.cursor() cur = context.db.cursor()
cur.execute( cur.execute(
"""insert into placex (osm_type, osm_id, class, type, name, admin_level, """insert into placex (osm_type, osm_id, class, type, name, admin_level,
housenumber, street, addr_place, isin, postcode, country_code, extratags, address, extratags, geometry)
geometry)
select * from place where not (class='place' and type='houses' and osm_type='W')""") select * from place where not (class='place' and type='houses' and osm_type='W')""")
cur.execute( cur.execute(
"""insert into location_property_osmline """insert into location_property_osmline (osm_id, address, linegeo)
(osm_id, interpolationtype, street, addr_place, SELECT osm_id, address, geometry from place
postcode, calculated_country_code, linegeo)
SELECT osm_id, housenumber, street, addr_place,
postcode, country_code, geometry from place
WHERE class='place' and type='houses' and osm_type='W' WHERE class='place' and type='houses' and osm_type='W'
and ST_GeometryType(geometry) = 'ST_LineString'""") and ST_GeometryType(geometry) = 'ST_LineString'""")
context.db.commit() context.db.commit()
@@ -338,7 +341,7 @@ def check_placex_contents(context, exact):
expected_content.add((res['osm_type'], res['osm_id'], res['class'])) expected_content.add((res['osm_type'], res['osm_id'], res['class']))
for h in row.headings: for h in row.headings:
msg = "%s: %s" % (row['object'], h) msg = "%s: %s" % (row['object'], h)
if h in ('name', 'extratags'): if h in ('name', 'extratags', 'address'):
if row[h] == '-': if row[h] == '-':
assert_is_none(res[h], msg) assert_is_none(res[h], msg)
else: else:
@@ -348,6 +351,12 @@ def check_placex_contents(context, exact):
assert_equals(res['name'][h[5:]], row[h], msg) assert_equals(res['name'][h[5:]], row[h], msg)
elif h.startswith('extratags+'): elif h.startswith('extratags+'):
assert_equals(res['extratags'][h[10:]], row[h], msg) assert_equals(res['extratags'][h[10:]], row[h], msg)
elif h.startswith('addr+'):
if row[h] == '-':
if res['address'] is not None:
assert_not_in(h[5:], res['address'])
else:
assert_equals(res['address'][h[5:]], row[h], msg)
elif h in ('linked_place_id', 'parent_place_id'): elif h in ('linked_place_id', 'parent_place_id'):
if row[h] == '0': if row[h] == '0':
assert_equals(0, res[h], msg) assert_equals(0, res[h], msg)

10
test/bdd/steps/osm_data.py
View File

@@ -79,14 +79,10 @@ def update_from_osm_file(context):
cur = context.db.cursor() cur = context.db.cursor()
cur.execute("""insert into placex (osm_type, osm_id, class, type, name, cur.execute("""insert into placex (osm_type, osm_id, class, type, name,
admin_level, housenumber, street, addr_place, isin, postcode, admin_level, address, extratags, geometry) select * from place""")
country_code, extratags, geometry) select * from place""")
cur.execute( cur.execute(
"""insert into location_property_osmline """insert into location_property_osmline (osm_id, address, linegeo)
(osm_id, interpolationtype, street, addr_place, SELECT osm_id, address, geometry from place
postcode, calculated_country_code, linegeo)
SELECT osm_id, housenumber, street, addr_place,
postcode, country_code, geometry from place
WHERE class='place' and type='houses' and osm_type='W' WHERE class='place' and type='houses' and osm_type='W'
and ST_GeometryType(geometry) = 'ST_LineString'""") and ST_GeometryType(geometry) = 'ST_LineString'""")
context.db.commit() context.db.commit()