prepare release 4.2.1

.github/actions/setup-postgresql/action.yml

@@ -15,7 +15,9 @@ runs:
         - name: Remove existing PostgreSQL
           run: |
               sudo apt-get purge -yq postgresql*
-              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+              sudo apt install curl ca-certificates gnupg
+              curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+              sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
 
           shell: bash

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
 set(NOMINATIM_VERSION_MINOR 2)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 1)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 

ChangeLog

@@ -1,3 +1,7 @@
+4.2.1
+
+ * fix XSS vulnerability in debug view
+
 4.2.0
 
  * add experimental support for osm2pgsql flex style
@@ -21,6 +25,10 @@
  * typing fixes to work with latest type annotations from typeshed
  * smaller improvements to documentation (thanks to @mausch)
 
+4.1.1
+
+ * fix XSS vulnerability in debug view
+
 4.1.0
 
  * switch to ICU tokenizer as default
@@ -57,6 +65,10 @@
  * add setup instructions for updates and systemd
  * drop support for PostgreSQL 9.5
 
+4.0.2
+
+ * fix XSS vulnerability in debug view
+
 4.0.1
 
  * fix initialisation error in replication script
@@ -95,6 +107,10 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
 3.7.2
 
  * fix database check for reverse-only imports

docs/admin/Import.md

@@ -74,7 +74,7 @@ but it will improve the quality of the results if this is installed.
 This data is available as a binary download. Put it into your project directory:
 
     cd $PROJECT_DIR
-    wget https://nominatim.org/data/wikimedia-importance.sql.gz
+    wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
 
 The file is about 400MB and adds around 4GB to the Nominatim database.
 
@@ -92,8 +92,8 @@ and the UK (using the [CodePoint OpenData set](https://osdatahub.os.uk/downloads
 This data can be optionally downloaded into the project directory:
 
     cd $PROJECT_DIR
-    wget https://nominatim.org/data/gb_postcodes.csv.gz
-    wget https://nominatim.org/data/us_postcodes.csv.gz
+    wget https://www.nominatim.org/data/gb_postcodes.csv.gz
+    wget https://www.nominatim.org/data/us_postcodes.csv.gz
 
 You can also add your own custom postcode sources, see
 [Customization of postcodes](../customize/Postcodes.md).

docs/admin/Installation.md

@@ -135,7 +135,7 @@ git clone --recursive https://github.com/openstreetmap/Nominatim.git
 The development version does not include the country grid. Download it separately:
 
 ```
-wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+wget -O Nominatim/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
 ```
 
 ### Building Nominatim

docs/admin/Migration.md

@@ -15,14 +15,6 @@ breaking changes. **Please read them before running the migration.**
     If you are migrating from a version <3.6, then you still have to follow
     the manual migration steps up to 3.6.
 
-## 4.2.2 -> 4.2.3
-
-### Update interpolation functions
-
-When updating to this release, you need to run `nominatim refresh --functions`
-after updating and before restarting updates. Otherwise you may see an error
-`Splitting of Point geometries is unsupported` or similar.
-
 ## 4.0.0 -> 4.1.0
 
 ### ICU tokenizer is the new default

docs/api/Faq.md

@@ -59,27 +59,3 @@ suited for these kinds of queries.
 
 That said if you installed your own Nominatim instance you can use the
 `nominatim export` PHP script as basis to return such lists.
-
-#### 7. My result has a wrong postcode. Where does it come from?
-
-Most places in OSM don't have a postcode, so Nominatim tries to interpolate
-one. It first look at all the places that make up the address of the place.
-If one of them has a postcode defined, this is the one to be used. When
-none of the address parts has a postcode either, Nominatim interpolates one
-from the surrounding objects. If the postcode is for your result is one, then
-most of the time there is an OSM object with the wrong postcode nearby.
-
-To find the bad postcode, go to
-[https://nominatim.openstreetmap.org](https://nominatim.openstreetmap.org)
-and search for your place. When you have found it, click on the 'details' link
-under the result to go to the details page. There is a field 'Computed Postcode'
-which should display the bad postcode. Click on the 'how?' link. A small
-explanation text appears. It contains a link to a query for Overpass Turbo.
-Click on that and you get a map with all places in the area that have the bad
-postcode. If none is displayed, zoom the map out a bit and then click on 'Run'.
-
-Now go to [OpenStreetMap](https://openstreetmap.org) and fix the error you
-have just found. It will take at least a day for Nominatim to catch up with
-your data fix. Sometimes longer, depending on how much editing activity is in
-the area.
-

docs/api/Output.md

@@ -211,8 +211,8 @@ be more than one. The attributes of that element contain:
  * `ref` - content of `ref` tag if it exists
  * `lat`, `lon` - latitude and longitude of the centroid of the object
  * `boundingbox` - comma-separated list of corner coordinates ([see notes](#boundingbox))
- * `place_rank` - class [search rank](../customize/Ranking.md#search-rank)
- * `address_rank` - place [address rank](../customize/Ranking.md#address-rank)
+ * `place_rank` - class [search rank](../customize/Ranking#search-rank)
+ * `address_rank` - place [address rank](../customize/Ranking#address-rank)
  * `display_name` - full comma-separated address
  * `class`, `type` - key and value of the main OSM tag
  * `importance` - computed importance rank

docs/api/Reverse.md

@@ -35,7 +35,7 @@ Additional parameters are accepted as listed below.
 
 !!! warning "Deprecation warning"
     The reverse API used to allow address lookup for a single OSM object by
-    its OSM id. This use is now deprecated. Use the [Address Lookup API](Lookup.md)
+    its OSM id. This use is now deprecated. Use the [Address Lookup API](../Lookup)
     instead.
 
 ### Output format

docs/customize/Importance.md

@@ -12,7 +12,7 @@ customize them.
 The main value for importance is derived from page ranking values for Wikipedia
 pages for a place. For places that do not have their own
 Wikipedia page, a formula is used that derives a static importance from the
-places [search rank](../customize/Ranking.md#search-rank).
+places [search rank](../customize/Ranking#search-rank).
 
 In a second step, a secondary importance value is added which is meant to
 represent how well-known the general area is where the place is located. It

docs/customize/Settings.md

@@ -666,7 +666,7 @@ The entries in the log file have the following format:
     <request time> <execution time in s> <number of results> <type> "<query string>"
 
 Request time is the time when the request was started. The execution time is
-given in seconds and corresponds to the time the query took executing in PHP.
+given in ms and corresponds to the time the query took executing in PHP.
 type contains the name of the endpoint used.
 
 Can be used as the same time as NOMINATIM_LOG_DB.

docs/extra.css

@@ -1,3 +1,7 @@
+.toctree-l3 {
+    display: none!important
+}
+
 table {
     margin-bottom: 12pt
 }

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim 4.2.4
+site_name: Nominatim Documentation
 theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org

lib-php/DebugHtml.php

@@ -135,7 +135,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -178,11 +178,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
-            return strlen($mVar) + 2;
+            $sOut = "'$mVar'";
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
-        return strlen((string)$mVar);
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
+        return strlen($sOut);
     }
 }

nominatim/version.py

@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 2, 0, 0)
+NOMINATIM_VERSION = (4, 2, 1, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

test/php/Nominatim/DebugTest.php

@@ -37,14 +37,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
 <pre><b>Var1:</b>  <i>True</i></pre>
 <pre><b>Var2:</b>  <i>False</i></pre>
 <pre><b>Var3:</b>  0</pre>
-<pre><b>Var4:</b>  'String'</pre>
-<pre><b>Var5:</b>  0 => 'one'
-       1 => 'two'
-       2 => 'three'</pre>
-<pre><b>Var6:</b>  'key' => 'value'
-       'key2' => 'value2'</pre>
+<pre><b>Var4:</b>  &#039;String&#039;</pre>
+<pre><b>Var5:</b>  0 => &#039;one&#039;
+       1 => &#039;two&#039;
+       2 => &#039;three&#039;</pre>
+<pre><b>Var6:</b>  &#039;key&#039; => &#039;value&#039;
+       &#039;key2&#039; => &#039;value2&#039;</pre>
 <pre><b>Var7:</b>  me as string</pre>
-<pre><b>Var8:</b>  'value', 'value2'</pre>
+<pre><b>Var8:</b>  &#039;value&#039;, &#039;value2&#039;</pre>
 
 EOT
         );
@@ -64,10 +64,10 @@ EOT
     public function testDebugArray()
     {
         $this->expectOutputString(<<<EOT
-<pre><b>Arr0:</b>  'null'</pre>
-<pre><b>Arr1:</b>  'key1' => 'val1'
-       'key2' => 'val2'
-       'key3' => 'val3'</pre>
+<pre><b>Arr0:</b>  &#039;null&#039;</pre>
+<pre><b>Arr1:</b>  &#039;key1&#039; => &#039;val1&#039;
+       &#039;key2&#039; => &#039;val2&#039;
+       &#039;key3&#039; => &#039;val3&#039;</pre>
 
 EOT
         );
@@ -93,12 +93,12 @@ EOT
     <th><small>1</small></th>
   </tr>
   <tr>
-    <td><pre>'one'</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
-    <td><pre>'three'</pre></td>
-    <td><pre>'four'</pre></td>
+    <td><pre>&#039;three&#039;</pre></td>
+    <td><pre>&#039;four&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -109,9 +109,9 @@ EOT
     <th><small>key3</small></th>
   </tr>
   <tr>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 
@@ -147,18 +147,18 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'one'</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group2</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -171,15 +171,15 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
-    <td><pre>'val2'</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 

vagrant/Install-on-Ubuntu-18.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-pip \
-                        python3-psutil python3-jinja2 python3-yaml python3-icu
+                        python3-psutil python3-jinja2 python3-yaml python3-icu git
 
 # Some of the Python packages that come with Ubuntu 18.04 are too old, so
 # install the latest version from pip:
@@ -105,18 +105,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
-    tar xf Nominatim-4.2.4.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.2.4
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-20.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie python3-yaml
+                        python3-icu python3-datrie python3-yaml git
 
 #
 # System Configuration
@@ -99,18 +99,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
-    tar xf Nominatim-4.2.4.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.2.4
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-22.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie
+                        python3-icu python3-datrie git
 
 #
 # System Configuration
@@ -99,18 +99,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
-    tar xf Nominatim-4.2.4.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.2.4
+    cmake $USERHOME/Nominatim
     make
     sudo make install