adapt osm2pgsql tests for new address column

test/bdd/environment.py

@@ -4,6 +4,7 @@ import os
 import psycopg2
 import psycopg2.extras
 import subprocess
+import tempfile
 from nose.tools import * # for assert functions
 from sys import version_info as python_version
 
@@ -88,17 +89,20 @@ class NominatimEnvironment(object):
         conn.commit()
         conn.close()
 
-        # execute osm2pgsql on an empty file to get the right tables
-        osm2pgsql = os.path.join(self.build_dir, 'osm2pgsql', 'osm2pgsql')
-        proc = subprocess.Popen([osm2pgsql, '-lsc', '-r', 'xml',
-                                 '-O', 'gazetteer', '-d', self.template_db, '-'],
-                                cwd=self.build_dir, stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        [outstr, errstr] = proc.communicate(input=b'<osm version="0.6"></osm>')
-        logger.debug("running osm2pgsql for template: %s\n%s\n%s" % (osm2pgsql, outstr, errstr))
-        self.run_setup_script('create-functions', 'create-tables',
-                              'create-partition-tables', 'create-partition-functions',
-                              'load-data', 'create-search-indices')
+        # execute osm2pgsql import on an empty file to get the right tables
+        with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
+            fd.write(b'<osm version="0.6"></osm>')
+            fd.flush()
+            self.run_setup_script('import-data',
+                                  'ignore-errors',
+                                  'create-functions',
+                                  'create-tables',
+                                  'create-partition-tables',
+                                  'create-partition-functions',
+                                  'load-data',
+                                  'create-search-indices',
+                                  osm_file=fd.name,
+                                  osm2pgsql_cache='200')
 
     def setup_api_db(self, context):
         self.write_nominatim_config(self.api_test_db)

test/bdd/osm2pgsql/import/tags.feature

@@ -102,8 +102,8 @@ Feature: Tag evaluation
          n1 T<key>=<value>,name=real
          """
         Then place contains
-         | object | name |
-         | N1     | 'name' : 'real' |
+         | object | class | type    | name |
+         | N1     | <key> | <value> | 'name' : 'real' |
 
     Examples:
      | key       | value |
@@ -300,7 +300,7 @@ Feature: Tag evaluation
           n11 T<key>=<value>
           """
         Then place contains
-          | object | class   | type      | postcode |
+          | object | class   | type      | addr+postcode |
           | N10    | highway | secondary | <value> |
           | N11    | place   | postcode  | <value> |
         And place has no entry for N10:place
@@ -320,9 +320,9 @@ Feature: Tag evaluation
           n20 Tamenity=hospital,addr:place=Foo%20%Town
           """
         Then place contains
-          | object | class   | type     | street  | addr_place |
-          | N10    | amenity | hospital | Foo St  | None |
-          | N20    | amenity | hospital | -       | Foo Town |
+          | object | class   | type     | addr+street | addr+place |
+          | N10    | amenity | hospital | Foo St      | -        |
+          | N20    | amenity | hospital | -           | Foo Town |
 
 
     Scenario Outline: Import of country
@@ -331,16 +331,16 @@ Feature: Tag evaluation
           n10 Tplace=village,<key>=<value>
           """
         Then place contains
-          | object | class   | type    | country_code |
+          | object | class   | type    | addr+country |
           | N10    | place   | village | <value> |
 
     Examples:
-        | key                            | value |
-        | country_code                   | us |
-        | ISO3166-1                      | XX |
-        | is_in:country_code          | __ |
-        | addr:country                | .. |
-        | addr:country_code           | cv |
+        | key                  | value |
+        | country_code         | us |
+        | ISO3166-1            | XX |
+        | is_in:country_code   | __ |
+        | addr:country         | .. |
+        | addr:country_code    | cv |
 
     Scenario Outline: Ignore country codes with wrong length
         When loading osm data
@@ -348,7 +348,7 @@ Feature: Tag evaluation
           n10 Tplace=village,country_code=<value>
           """
         Then place contains
-          | object | class   | type    | country_code |
+          | object | class   | type    | addr+country |
           | N10    | place   | village | - |
 
     Examples:
@@ -368,24 +368,11 @@ Feature: Tag evaluation
           n13 Tbuilding=yes,addr:conscriptionnumber=3,addr:streetnumber=111
           """
         Then place contains
-          | object | class | type   | housenumber |
-          | N10    | building | yes  | 4b |
-          | N11    | building | yes  | 003 |
-          | N12    | building | yes  | 2345 |
-          | N13    | building | yes  | 3/111 |
-
-    Scenario: Import of address interpolations
-        When loading osm data
-          """
-          n10 Taddr:interpolation=odd
-          n11 Taddr:housenumber=10,addr:interpolation=odd
-          n12 Taddr:interpolation=odd,addr:housenumber=23
-          """
-        Then place contains
-          | object | class   | type    | housenumber |
-          | N10    | place   | houses  | odd |
-          | N11    | place   | houses  | odd |
-          | N12    | place   | houses  | odd |
+          | object | class | type    | address |
+          | N10    | building | yes  | 'housenumber' : '4b' |
+          | N11    | building | yes  | 'conscriptionnumber' : '003' |
+          | N12    | building | yes  | 'streetnumber' : '2345' |
+          | N13    | building | yes  | 'conscriptionnumber' : '3', 'streetnumber' : '111' |
 
     Scenario: Shorten tiger:county tags
         When loading osm data
@@ -395,26 +382,26 @@ Feature: Tag evaluation
           n12 Tplace=village,tiger:county=Feebourgh
           """
         Then place contains
-          | object | class   | type    | isin |
+          | object | class   | type    | addr+tiger:county |
           | N10    | place   | village | Feebourgh county |
-          | N11    | place   | village | Alabama,Feebourgh county |
+          | N11    | place   | village | Feebourgh county |
           | N12    | place   | village | Feebourgh county |
 
     Scenario Outline: Import of address tags
         When loading osm data
           """
-          n10 Tplace=village,<key>=<value>
+          n10 Tplace=village,addr:<key>=<value>
+          n11 Tplace=village,is_in:<key>=<value>
           """
         Then place contains
-          | object | class   | type    | isin |
-          | N10    | place   | village | <value> |
+          | object | class   | type    | address |
+          | N10    | place   | village | '<key>' : '<value>' |
 
     Examples:
-      | key             | value |
-      | is_in:country   | Xanadu |
-      | addr:suburb     | hinein |
-      | addr:city       | Sydney |
-      | addr:state      | Jura |
+      | key       | value |
+      | suburb    | hinein |
+      | city      | Sydney |
+      | state     | Jura |
 
     Scenario: Import of isin tags with space
         When loading osm data
@@ -423,9 +410,9 @@ Feature: Tag evaluation
           n11 Tplace=village,addr:county=le%20%havre
           """
         Then place contains
-          | object | class   | type    | isin |
-          | N10    | place   | village | Stockholm, Sweden |
-          | N11    | place   | village | le havre |
+          | object | class   | type    | address |
+          | N10    | place   | village | 'is_in' : 'Stockholm, Sweden' |
+          | N11    | place   | village | 'county' : 'le havre' |
 
     Scenario: Import of admin level
         When loading osm data
@@ -438,8 +425,8 @@ Feature: Tag evaluation
         Then place contains
           | object | class   | type     | admin_level |
           | N10    | amenity | hospital | 3 |
-          | N11    | amenity | hospital | 100 |
-          | N12    | amenity | hospital | 100 |
+          | N11    | amenity | hospital | 15 |
+          | N12    | amenity | hospital | 15 |
           | N13    | amenity | hospital | 3 |
 
     Scenario Outline: Import of extra tags
@@ -550,5 +537,5 @@ Feature: Tag evaluation
           n290393920 Taddr:city=Perpignan,addr:country=FR,addr:housenumber=43\,addr:postcode=66000,addr:street=Rue%20%Pierre%20%Constant%20%d`Ivry,source=cadastre-dgi-fr%20%source%20%:%20%Direction%20%Générale%20%des%20%Impôts%20%-%20%Cadastre%20%;%20%mise%20%à%20%jour%20%:2008
           """
         Then place contains
-         | object     | class   | type | housenumber |
-         | N290393920 | place   | house| 43\ |
+         | object     | class   | type | address |
+         | N290393920 | place   | house| 'city' : 'Perpignan', 'country' : 'FR', 'housenumber' : '43\\', 'postcode' : '66000', 'street' : 'Rue Pierre Constant d`Ivry' |

test/bdd/osm2pgsql/update/relation.feature

@@ -137,7 +137,7 @@ Feature: Update of relations by osm2pgsql
           r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=XX,admin_level=2 Mw1@
           """
         Then place contains
-          | object | country_code | name           |
+          | object | addr+country | name           |
           | R1     | XX           | 'name' : 'Foo' |
 
     Scenario: Country boundary names are extended when country_code known
@@ -154,6 +154,6 @@ Feature: Update of relations by osm2pgsql
           r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=ch,admin_level=2 Mw1@
           """
         Then place contains
-            | object | country_code | name+name:de | name+name |
+            | object | addr+country | name+name:de | name+name |
             | R1     | ch           | Schweiz      | Foo       |
 

test/bdd/steps/db_ops.py

@@ -20,9 +20,10 @@ class PlaceColumn:
             self.add_hstore('name', key[5:], value)
         elif key.startswith('extra+'):
             self.add_hstore('extratags', key[6:], value)
+        elif key.startswith('addr+'):
+            self.add_hstore('address', key[6:], value)
         else:
-            assert_in(key, ('class', 'type', 'street', 'addr_place',
-                            'isin', 'postcode'))
+            assert_in(key, ('class', 'type'))
             self.columns[key] = None if value == '' else value
 
     def set_key_name(self, value):
@@ -39,10 +40,16 @@ class PlaceColumn:
         self.columns['admin_level'] = int(value)
 
     def set_key_housenr(self, value):
-        self.columns['housenumber'] = None if value == '' else value
+        self.add_hstore('address', 'housenumber', None if value == '' else value)
+
+    def set_key_street(self, value):
+        self.add_hstore('address', 'street', None if value == '' else value)
+
+    def set_key_addr_place(self, value):
+        self.add_hstore('address', 'place', None if value == '' else value)
 
     def set_key_country(self, value):
-        self.columns['country_code'] = None if value == '' else value
+        self.add_hstore('address', 'country', None if value == '' else value)
 
     def set_key_geometry(self, value):
         self.geometry = self.context.osm.parse_geometry(value, self.context.scene)
@@ -223,15 +230,11 @@ def import_and_index_data_from_place_table(context):
     cur = context.db.cursor()
     cur.execute(
         """insert into placex (osm_type, osm_id, class, type, name, admin_level,
-           housenumber, street, addr_place, isin, postcode, country_code, extratags,
-           geometry)
+           address, extratags, geometry)
            select * from place where not (class='place' and type='houses' and osm_type='W')""")
     cur.execute(
-            """insert into location_property_osmline
-               (osm_id, interpolationtype, street, addr_place,
-                postcode, calculated_country_code, linegeo)
-             SELECT osm_id, housenumber, street, addr_place,
-                    postcode, country_code, geometry from place
+            """insert into location_property_osmline (osm_id, address, linegeo)
+             SELECT osm_id, address, geometry from place
               WHERE class='place' and type='houses' and osm_type='W'
                     and ST_GeometryType(geometry) = 'ST_LineString'""")
     context.db.commit()
@@ -338,7 +341,7 @@ def check_placex_contents(context, exact):
                 expected_content.add((res['osm_type'], res['osm_id'], res['class']))
             for h in row.headings:
                 msg = "%s: %s" % (row['object'], h)
-                if h in ('name', 'extratags'):
+                if h in ('name', 'extratags', 'address'):
                     if row[h] == '-':
                         assert_is_none(res[h], msg)
                     else:
@@ -348,6 +351,12 @@ def check_placex_contents(context, exact):
                     assert_equals(res['name'][h[5:]], row[h], msg)
                 elif h.startswith('extratags+'):
                     assert_equals(res['extratags'][h[10:]], row[h], msg)
+                elif h.startswith('addr+'):
+                    if row[h] == '-':
+                        if res['address']  is not None:
+                            assert_not_in(h[5:], res['address'])
+                    else:
+                        assert_equals(res['address'][h[5:]], row[h], msg)
                 elif h in ('linked_place_id', 'parent_place_id'):
                     if row[h] == '0':
                         assert_equals(0, res[h], msg)

test/bdd/steps/osm_data.py

@@ -79,14 +79,10 @@ def update_from_osm_file(context):
 
     cur = context.db.cursor()
     cur.execute("""insert into placex (osm_type, osm_id, class, type, name,
-                   admin_level,  housenumber, street, addr_place, isin, postcode,
-                   country_code, extratags, geometry) select * from place""")
+                   admin_level, address, extratags, geometry) select * from place""")
     cur.execute(
-        """insert into location_property_osmline
-               (osm_id, interpolationtype, street, addr_place,
-                postcode, calculated_country_code, linegeo)
-             SELECT osm_id, housenumber, street, addr_place,
-                    postcode, country_code, geometry from place
+        """insert into location_property_osmline (osm_id, address, linegeo)
+             SELECT osm_id, address, geometry from place
               WHERE class='place' and type='houses' and osm_type='W'
                     and ST_GeometryType(geometry) = 'ST_LineString'""")
     context.db.commit()