prepare release 4.0.2

Don't penalize French 'bis' housenumbers

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
 set(NOMINATIM_VERSION_MINOR 0)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 2)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 

ChangeLog

@@ -1,3 +1,16 @@
+4.0.2
+
+ * fix XSS vulnerability in debug view
+
+4.0.1
+
+ * fix initialisation error in replication script
+ * ICU tokenizer: avoid any special characters in word tokens
+ * better error message when API php script does not exist
+ * fix quoting of house numbers in SQL queries
+ * small fixes and improvements in search query parsing
+ * add documentation for moving the database to a different machine
+
 4.0.0
 
  * refactor name token computation and introduce ICU tokenizer
@@ -27,6 +40,10 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
 3.7.2
 
  * fix database check for reverse-only imports

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim 4.0.1
+site_name: Nominatim Documentation
 theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org

lib-php/DebugHtml.php

@@ -127,7 +127,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -170,11 +170,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
-            return strlen($mVar) + 2;
+            $sOut = "'$mVar'";
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
-        return strlen((string)$mVar);
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
+        return strlen($sOut);
     }
 }

lib-php/SearchDescription.php

@@ -257,7 +257,7 @@ class SearchDescription
         if (empty($this->aName)) {
             $this->bNameNeedsAddress = $bNeedsAddress;
         } else {
-            $this->bNameNeedsAddress |= $bNeedsAddress;
+            $this->bNameNeedsAddress &= $bNeedsAddress;
         }
         if ($bSearchable) {
             $this->aName[$iId] = $iId;
@@ -584,11 +584,11 @@ class SearchDescription
         // will be narrowed down by an address. Remember that with ordering
         // every single result has to be checked.
         if ($this->sHouseNumber && ($this->bRareName || !empty($this->aAddress) || $this->sPostcode)) {
-            $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
+            $sHouseNumberRegex = $oDB->getDBQuoted('\\\\m'.$this->sHouseNumber.'\\\\M');
 
             // Housenumbers on streets and places.
             $sChildHnr = 'SELECT * FROM placex WHERE parent_place_id = search_name.place_id';
-            $sChildHnr .= "    AND housenumber ~* E'".$sHouseNumberRegex."'";
+            $sChildHnr .= '    AND housenumber ~* E'.$sHouseNumberRegex;
             // Interpolations on streets and places.
             if (preg_match('/^[0-9]+$/', $this->sHouseNumber)) {
                 $sIpolHnr = 'SELECT * FROM location_property_osmline ';
@@ -601,7 +601,7 @@ class SearchDescription
             }
             // Housenumbers on the object iteself for unlisted places.
             $sSelfHnr = 'SELECT * FROM placex WHERE place_id = search_name.place_id';
-            $sSelfHnr .= "    AND housenumber ~* E'".$sHouseNumberRegex."'";
+            $sSelfHnr .= '    AND housenumber ~* E'.$sHouseNumberRegex;
 
             $sSql = '(CASE WHEN address_rank = 30 THEN EXISTS('.$sSelfHnr.') ';
             $sSql .= ' ELSE EXISTS('.$sChildHnr.') ';
@@ -739,9 +739,9 @@ class SearchDescription
             return $aResults;
         }
 
-        $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
+        $sHouseNumberRegex = $oDB->getDBQuoted('\\\\m'.$this->sHouseNumber.'\\\\M');
         $sSQL = 'SELECT place_id FROM placex WHERE';
-        $sSQL .= "  housenumber ~* E'".$sHouseNumberRegex."'";
+        $sSQL .= '  housenumber ~* E'.$sHouseNumberRegex;
         $sSQL .= ' AND ('.join(' OR ', $aIDCondition).')';
         $sSQL .= $this->oContext->excludeSQL(' AND place_id');
 

lib-php/TokenHousenumber.php

@@ -58,7 +58,7 @@ class HouseNumber
         // up of numbers, add a penalty
         $iSearchCost = 1;
         if (preg_match('/\\d/', $this->sToken) === 0
-            || preg_match_all('/[^0-9]/', $this->sToken, $aMatches) > 2) {
+            || preg_match_all('/[^0-9 ]/', $this->sToken, $aMatches) > 3) {
             $iSearchCost += strlen($this->sToken) - 1;
         }
         if (!$oSearch->hasOperator(\Nominatim\Operator::NONE)) {

nominatim/clicmd/api.py

@@ -4,6 +4,7 @@ Subcommand definitions for API calls from the command line.
 import logging
 
 from nominatim.tools.exec_utils import run_api_script
+from nominatim.errors import UsageError
 
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
@@ -53,6 +54,18 @@ def _add_api_output_arguments(parser):
                              "Parameter is difference tolerance in degrees."))
 
 
+def _run_api(endpoint, args, params):
+    script_file = args.project_dir / 'website' / (endpoint + '.php')
+
+    if not script_file.exists():
+        LOG.error("Cannot find API script file.\n\n"
+                  "Make sure to run 'nominatim' from the project directory \n"
+                  "or use the option --project-dir.")
+        raise UsageError("API script not found.")
+
+    return run_api_script(endpoint, args.project_dir,
+                          phpcgi_bin=args.phpcgi_path, params=params)
+
 class APISearch:
     """\
     Execute a search query.
@@ -114,8 +127,7 @@ class APISearch:
         if not args.dedupe:
             params['dedupe'] = '0'
 
-        return run_api_script('search', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
+        return _run_api('search', args, params)
 
 class APIReverse:
     """\
@@ -158,8 +170,7 @@ class APIReverse:
         if args.polygon_threshold:
             params['polygon_threshold'] = args.polygon_threshold
 
-        return run_api_script('reverse', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
+        return _run_api('reverse', args, params)
 
 
 class APILookup:
@@ -198,8 +209,7 @@ class APILookup:
         if args.polygon_threshold:
             params['polygon_threshold'] = args.polygon_threshold
 
-        return run_api_script('lookup', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
+        return _run_api('lookup', args, params)
 
 
 class APIDetails:
@@ -249,8 +259,7 @@ class APIDetails:
         for name, _ in DETAILS_SWITCHES:
             params[name] = '1' if getattr(args, name) else '0'
 
-        return run_api_script('details', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
+        return _run_api('details', args, params)
 
 
 class APIStatus:
@@ -271,6 +280,4 @@ class APIStatus:
 
     @staticmethod
     def run(args):
-        return run_api_script('status', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path,
-                              params=dict(format=args.format))
+        return _run_api('status', args, dict(format=args.format))

nominatim/clicmd/replication.py

@@ -136,6 +136,7 @@ class UpdateReplication:
             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 
         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
+        indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)
 
         while True:
             with connect(args.config.get_libpq_dsn()) as conn:
@@ -148,8 +149,6 @@ class UpdateReplication:
 
             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
                 index_start = dt.datetime.now(dt.timezone.utc)
-                indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
-                                  args.threads or 1)
                 indexer.index_full(analyse=False)
 
                 with connect(args.config.get_libpq_dsn()) as conn:

nominatim/version.py

@@ -10,7 +10,7 @@ Version information for Nominatim.
 # and must always be increased when there is a change to the database or code
 # that requires a migration.
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 0, 0, 0)
+NOMINATIM_VERSION = (4, 0, 2, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 5)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/icu_tokenizer.yaml

@@ -21,8 +21,8 @@ transliteration:
     - !include icu-rules/extended-unicode-to-asccii.yaml
     - ":: Ascii ()"
     - ":: NFD ()"
-    - "[^[:Ascii:]] >"
     - ":: lower ()"
+    - "[^a-z0-9[:Space:]] >"
     - ":: NFC ()"
 sanitizers:
     - step: split-name-list

test/php/Nominatim/DebugTest.php

@@ -29,14 +29,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
 <pre><b>Var1:</b>  <i>True</i></pre>
 <pre><b>Var2:</b>  <i>False</i></pre>
 <pre><b>Var3:</b>  0</pre>
-<pre><b>Var4:</b>  'String'</pre>
-<pre><b>Var5:</b>  0 => 'one'
-       1 => 'two'
-       2 => 'three'</pre>
-<pre><b>Var6:</b>  'key' => 'value'
-       'key2' => 'value2'</pre>
+<pre><b>Var4:</b>  &#039;String&#039;</pre>
+<pre><b>Var5:</b>  0 => &#039;one&#039;
+       1 => &#039;two&#039;
+       2 => &#039;three&#039;</pre>
+<pre><b>Var6:</b>  &#039;key&#039; => &#039;value&#039;
+       &#039;key2&#039; => &#039;value2&#039;</pre>
 <pre><b>Var7:</b>  me as string</pre>
-<pre><b>Var8:</b>  'value', 'value2'</pre>
+<pre><b>Var8:</b>  &#039;value&#039;, &#039;value2&#039;</pre>
 
 EOT
         );
@@ -56,10 +56,10 @@ EOT
     public function testDebugArray()
     {
         $this->expectOutputString(<<<EOT
-<pre><b>Arr0:</b>  'null'</pre>
-<pre><b>Arr1:</b>  'key1' => 'val1'
-       'key2' => 'val2'
-       'key3' => 'val3'</pre>
+<pre><b>Arr0:</b>  &#039;null&#039;</pre>
+<pre><b>Arr1:</b>  &#039;key1&#039; => &#039;val1&#039;
+       &#039;key2&#039; => &#039;val2&#039;
+       &#039;key3&#039; => &#039;val3&#039;</pre>
 
 EOT
         );
@@ -85,12 +85,12 @@ EOT
     <th><small>1</small></th>
   </tr>
   <tr>
-    <td><pre>'one'</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
-    <td><pre>'three'</pre></td>
-    <td><pre>'four'</pre></td>
+    <td><pre>&#039;three&#039;</pre></td>
+    <td><pre>&#039;four&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -101,9 +101,9 @@ EOT
     <th><small>key3</small></th>
   </tr>
   <tr>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 
@@ -139,18 +139,18 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'one'</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group2</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -163,15 +163,15 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 

test/python/test_cli.py

@@ -113,23 +113,36 @@ class TestCli:
         assert func.called == 1
 
 
-    @pytest.mark.parametrize("params", [('search', '--query', 'new'),
-                                        ('reverse', '--lat', '0', '--lon', '0'),
-                                        ('lookup', '--id', 'N1'),
-                                        ('details', '--node', '1'),
-                                        ('details', '--way', '1'),
-                                        ('details', '--relation', '1'),
-                                        ('details', '--place_id', '10001'),
-                                        ('status',)])
-    def test_api_commands_simple(self, mock_func_factory, params):
+@pytest.mark.parametrize("params", [('search', '--query', 'new'),
+                                    ('reverse', '--lat', '0', '--lon', '0'),
+                                    ('lookup', '--id', 'N1'),
+                                    ('details', '--node', '1'),
+                                    ('details', '--way', '1'),
+                                    ('details', '--relation', '1'),
+                                    ('details', '--place_id', '10001'),
+                                    ('status',)])
+class TestCliApiCall:
+
+    @pytest.fixture(autouse=True)
+    def setup_cli_call(self, cli_call):
+        self.call_nominatim = cli_call
+
+    def test_api_commands_simple(self, mock_func_factory, params, tmp_path):
+        (tmp_path / 'website').mkdir()
+        (tmp_path / 'website' / (params[0] + '.php')).write_text('')
         mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
 
-        assert self.call_nominatim(*params) == 0
+        assert self.call_nominatim(*params, '--project-dir', str(tmp_path)) == 0
 
         assert mock_run_api.called == 1
         assert mock_run_api.last_args[0] == params[0]
 
 
+    def test_bad_project_idr(self, mock_func_factory, params):
+        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
+
+        assert self.call_nominatim(*params) == 1
+
 
 class TestCliWithDb:
 

test/python/test_cli_replication.py

@@ -106,6 +106,16 @@ class TestCliReplication:
         assert str(update_mock.last_args[1]['osm2pgsql']) == '/secret/osm2pgsql'
 
 
+    @pytest.mark.parametrize("update_interval", [60, 3600])
+    def test_replication_catchup(self, monkeypatch, index_mock, update_interval, placex_table):
+        monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', str(update_interval))
+        states = [nominatim.tools.replication.UpdateState.NO_CHANGES]
+        monkeypatch.setattr(nominatim.tools.replication, 'update',
+                            lambda *args, **kwargs: states.pop())
+
+        assert self.call_nominatim('--catch-up') == 0
+
+
     def test_replication_update_custom_threads(self, update_mock):
         assert self.call_nominatim('--once', '--no-index', '--threads', '4') == 0
 

vagrant/Install-on-Centos-7.sh

@@ -35,12 +35,12 @@
 
     sudo yum install -y postgresql11-server postgresql11-contrib postgresql11-devel \
                         postgis25_11 postgis25_11-utils \
-                        wget cmake make gcc gcc-c++ libtool policycoreutils-python \
+                        wget git cmake make gcc gcc-c++ libtool policycoreutils-python \
                         devtoolset-7 llvm-toolset-7 \
                         php-pgsql php php-intl libpqxx-devel \
                         proj-epsg bzip2-devel proj-devel boost-devel \
                         python3-pip python3-setuptools python3-devel \
-                        expat-devel zlib-devel libicu-devel
+                        expat-devel zlib-devel libicu-dev
 
     pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml
 
@@ -117,19 +117,26 @@ sudo chown vagrant /srv/nominatim  #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.0.1.tar.bz2
-    tar xf Nominatim-4.0.1.tar.bz2
+    git clone --recursive git://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
 #DOCS:    :::sh
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.0.1
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Centos-8.sh

@@ -30,7 +30,7 @@
 #DOCS:    :::sh
     sudo dnf --enablerepo=powertools install -y postgresql12-server \
                         postgresql12-contrib postgresql12-devel postgis30_12 \
-                        wget cmake make gcc gcc-c++ libtool policycoreutils-python-utils \
+                        wget git cmake make gcc gcc-c++ libtool policycoreutils-python-utils \
                         llvm-toolset ccache clang-tools-extra \
                         php-pgsql php php-intl php-json libpq-devel \
                         bzip2-devel proj-devel boost-devel \
@@ -125,19 +125,26 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.0.1.tar.bz2
-    tar xf Nominatim-4.0.1.tar.bz2
+    git clone --recursive git://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
 #DOCS:    :::sh
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.0.1
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-18.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-server-dev-10 postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php php-pgsql php-intl libicu-dev python3-pip \
-                        python3-psutil python3-jinja2 python3-icu
+                        python3-psutil python3-jinja2 python3-icu git
 
 # Some of the Python packages that come with Ubuntu 18.04 are too old, so
 # install the latest version from pip:
@@ -105,18 +105,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.0.1.tar.bz2
-    tar xf Nominatim-4.0.1.tar.bz2
+    git clone --recursive git://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.0.1
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-20.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie
+                        python3-icu python3-datrie git
 
 #
 # System Configuration
@@ -99,8 +99,8 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.0.1.tar.bz2
-    tar xf Nominatim-4.0.1.tar.bz2
+    git clone --recursive git://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
@@ -117,7 +117,7 @@ fi                                 #DOCS:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.0.1
+    cmake $USERHOME/Nominatim
     make
     sudo make install