Merge pull request #3122 from miku0/sanitizer-final

Adds sanitizer for Japanese addresses to correspond to block address

test/bdd/db/query/japanese.feature

@@ -0,0 +1,30 @@
+@DB
+Feature: Searches in Japan
+    Test specifically for searches of Japanese addresses and in Japanese language.
+    @fail-legacy
+    Scenario: A block house-number is parented to the neighbourhood
+        Given the grid with origin JP
+          | 1 |   |   |   | 2 |
+          |   | 3 |   |   |   |
+          |   |   | 9 |   |   |
+          |   |   |   | 6 |   |
+        And the places
+          | osm | class   | type        | name       | geometry |
+          | W1  | highway | residential | 雉子橋通り | 1,2      |
+        And the places
+          | osm | class   | type       | housenr | addr+block_number | addr+neighbourhood | geometry |
+          | N3  | amenity | restaurant | 2       | 6                 | 2丁目              | 3        |
+        And the places
+          | osm | class | type          | name  | geometry |
+          | N9  | place | neighbourhood | 2丁目 | 9        |
+        And the places
+          | osm | class | type    | name | geometry |
+          | N6  | place | quarter | 加瀬 | 6        |
+        When importing
+        Then placex contains
+          | object | parent_place_id |
+          | N3     | N9              |
+        When sending search query "2丁目 6-2"
+        Then results contain
+          | osm |
+          | N3  |

test/python/tokenizer/sanitizers/test_tag_japanese.py

@@ -0,0 +1,80 @@
+from nominatim.data.place_info import PlaceInfo
+from nominatim.data.place_name import PlaceName
+from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
+from typing import Mapping, Optional, List
+import pytest
+
+class TestTagJapanese:
+    @pytest.fixture(autouse=True)
+    def setup_country(self, def_config):
+        self.config = def_config
+
+    def run_sanitizer_on(self,type, **kwargs):
+        place = PlaceInfo({
+            'address': kwargs,
+            'country_code': 'jp'
+        })
+        sanitizer_args = {'step': 'tag-japanese'}
+        _, address = PlaceSanitizer([sanitizer_args], self.config).process_names(place)
+        tmp_list = [(p.name,p.kind) for p in address]
+        return sorted(tmp_list)
+
+    def test_on_address(self):
+        res = self.run_sanitizer_on('address', name='foo', ref='bar', ref_abc='baz')
+        assert res == [('bar','ref'),('baz','ref_abc'),('foo','name')]
+
+    def test_housenumber(self):
+        res = self.run_sanitizer_on('address', housenumber='2')
+        assert res == [('2','housenumber')]
+
+    def test_blocknumber(self):
+        res = self.run_sanitizer_on('address', block_number='6')
+        assert res == [('6','housenumber')]
+
+    def test_neighbourhood(self):
+        res = self.run_sanitizer_on('address', neighbourhood='8')
+        assert res == [('8','place')]
+
+    def test_quarter(self):
+        res = self.run_sanitizer_on('address', quarter='kase')
+        assert res==[('kase','place')]
+
+    def test_housenumber_blocknumber(self):
+        res = self.run_sanitizer_on('address', housenumber='2', block_number='6')
+        assert res == [('6-2','housenumber')]
+
+    def test_quarter_neighbourhood(self):
+        res = self.run_sanitizer_on('address', quarter='kase', neighbourhood='8')
+        assert res == [('kase8','place')]
+
+    def test_blocknumber_housenumber_quarter(self):
+        res = self.run_sanitizer_on('address', block_number='6', housenumber='2', quarter='kase')
+        assert res == [('6-2','housenumber'),('kase','place')]
+
+    def test_blocknumber_housenumber_quarter_neighbourhood(self):
+        res = self.run_sanitizer_on('address', block_number='6', housenumber='2', neighbourhood='8')
+        assert res == [('6-2','housenumber'),('8','place')]
+
+    def test_blocknumber_quarter_neighbourhood(self):
+        res = self.run_sanitizer_on('address',block_number='6', quarter='kase', neighbourhood='8')
+        assert res == [('6','housenumber'),('kase8','place')]
+
+    def test_blocknumber_quarter(self):
+        res = self.run_sanitizer_on('address',block_number='6', quarter='kase')
+        assert res == [('6','housenumber'),('kase','place')]
+
+    def test_blocknumber_neighbourhood(self):
+        res = self.run_sanitizer_on('address',block_number='6', neighbourhood='8')
+        assert res == [('6','housenumber'),('8','place')]
+
+    def test_housenumber_quarter_neighbourhood(self):
+        res = self.run_sanitizer_on('address',housenumber='2', quarter='kase', neighbourhood='8')
+        assert res == [('2','housenumber'),('kase8','place')]
+
+    def test_housenumber_quarter(self):
+        res = self.run_sanitizer_on('address',housenumber='2', quarter='kase')
+        assert res == [('2','housenumber'),('kase','place')]
+
+    def test_housenumber_blocknumber_neighbourhood_quarter(self):
+        res = self.run_sanitizer_on('address', block_number='6', housenumber='2', quarter='kase', neighbourhood='8')
+        assert res == [('6-2','housenumber'),('kase8','place')]