Merge pull request #2197 from lonvia/use-jinja-for-sql-preprocessing

Use jinja2 for SQL preprocessing

.github/actions/build-nominatim/action.yml

@@ -6,7 +6,7 @@ runs:
     steps:
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2
           shell: bash
 
         - name: Download dependencies

docs/admin/Installation.md

@@ -42,6 +42,7 @@ For running Nominatim:
   * [Psycopg2](https://www.psycopg.org) (2.7+)
   * [Python Dotenv](https://github.com/theskumar/python-dotenv)
   * [psutil](https://github.com/giampaolo/psutil)
+  * [Jinja2](https://palletsprojects.com/p/jinja/)
   * [PHP](https://php.net) (7.0 or later)
   * PHP-pgsql
   * PHP-intl (bundled with PHP)

lib-php/admin/setup.php

@@ -123,24 +123,28 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
 
 if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->createFunctions();
+    $oSetup->createSqlFunctions();
 }
 
 if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->createTables($aCMDResult['reverse-only']);
+    $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables');
-    $oSetup->createFunctions();
+
-    $oSetup->createTableTriggers();
+    if ($aCMDResult['reverse-only'] ?? false) {
+        $oCmd->addParams('--reverse-only');
+    }
+
+    run($oCmd);
 }
 
 if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->createPartitionTables();
+    run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables'));
 }
 
 if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->createFunctions(); // also create partition functions
+    $oSetup->createSqlFunctions(); // also create partition functions
 }
 
 if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
@@ -182,7 +186,14 @@ if ($aCMDResult['drop']) {
 
 if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->createSearchIndices();
+
+    $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices');
+
+    if ($aCMDResult['drop'] ?? false) {
+        $oCmd->addParams('--drop');
+    }
+
+    run($oCmd);
 }
 
 if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {

lib-php/setup/SetupClass.php

@@ -67,52 +67,6 @@ class SetupFunctions
         }
     }
 
-    public function createFunctions()
-    {
-        info('Create Functions');
-
-        // Try accessing the C module, so we know early if something is wrong
-        $this->checkModulePresence(); // raises exception on failure
-
-        $this->createSqlFunctions();
-    }
-
-    public function createTables($bReverseOnly = false)
-    {
-        info('Create Tables');
-
-        $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql');
-        $sTemplate = $this->replaceSqlPatterns($sTemplate);
-
-        $this->pgsqlRunScript($sTemplate, false);
-
-        if ($bReverseOnly) {
-            $this->dropTable('search_name');
-        }
-
-        (clone($this->oNominatimCmd))->addParams('refresh', '--address-levels')->run();
-    }
-
-    public function createTableTriggers()
-    {
-        info('Create Tables');
-
-        $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql');
-        $sTemplate = $this->replaceSqlPatterns($sTemplate);
-
-        $this->pgsqlRunScript($sTemplate, false);
-    }
-
-    public function createPartitionTables()
-    {
-        info('Create Partition Tables');
-
-        $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql');
-        $sTemplate = $this->replaceSqlPatterns($sTemplate);
-
-        $this->pgsqlRunPartitionScript($sTemplate);
-    }
-
     public function importTigerData($sTigerPath)
     {
         info('Import Tiger data');
@@ -246,31 +200,6 @@ class SetupFunctions
         $this->db()->exec($sSQL);
     }
 
-    public function createSearchIndices()
-    {
-        info('Create Search indices');
-
-        $sSQL = 'SELECT relname FROM pg_class, pg_index ';
-        $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid';
-        $aInvalidIndices = $this->db()->getCol($sSQL);
-
-        foreach ($aInvalidIndices as $sIndexName) {
-            info("Cleaning up invalid index $sIndexName");
-            $this->db()->exec("DROP INDEX $sIndexName;");
-        }
-
-        $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql');
-        if (!$this->bDrop) {
-            $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql');
-        }
-        if (!$this->dbReverseOnly()) {
-            $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql');
-        }
-        $sTemplate = $this->replaceSqlPatterns($sTemplate);
-
-        $this->pgsqlRunScript($sTemplate);
-    }
-
     public function createCountryNames()
     {
         info('Create search index for default country names');
@@ -326,7 +255,7 @@ class SetupFunctions
         );
     }
 
-    private function createSqlFunctions()
+    public function createSqlFunctions()
     {
         $oCmd = (clone($this->oNominatimCmd))
                 ->addParams('refresh', '--functions');
@@ -342,24 +271,6 @@ class SetupFunctions
         $oCmd->run(!$this->sIgnoreErrors);
     }
 
-    private function pgsqlRunPartitionScript($sTemplate)
-    {
-        $sSQL = 'select distinct partition from country_name order by partition';
-        $aPartitions = $this->db()->getCol($sSQL);
-        if ($aPartitions[0] != 0) $aPartitions[] = 0;
-
-        preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
-        foreach ($aMatches as $aMatch) {
-            $sResult = '';
-            foreach ($aPartitions as $sPartitionName) {
-                $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
-            }
-            $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
-        }
-
-        $this->pgsqlRunScript($sTemplate);
-    }
-
     private function pgsqlRunScriptFile($sFilename)
     {
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
@@ -444,44 +355,4 @@ class SetupFunctions
 
         return $sSql;
     }
-
-    /**
-     * Drop table with the given name if it exists.
-     *
-     * @param string $sName Name of table to remove.
-     *
-     * @return null
-     */
-    private function dropTable($sName)
-    {
-        if ($this->bVerbose) echo "Dropping table $sName\n";
-        $this->db()->deleteTable($sName);
-    }
-
-    /**
-     * Check if the database is in reverse-only mode.
-     *
-     * @return True if there is no search_name table and infrastructure.
-     */
-    private function dbReverseOnly()
-    {
-        return !($this->db()->tableExists('search_name'));
-    }
-
-    /**
-     * Try accessing the C module, so we know early if something is wrong.
-     *
-     * Raises Nominatim\DatabaseError on failure
-     */
-    private function checkModulePresence()
-    {
-        $sModulePath = getSetting('DATABASE_MODULE_PATH', CONST_InstallDir.'/module');
-        $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
-        $sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
-        $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
-
-        $oDB = new \Nominatim\DB();
-        $oDB->connect();
-        $oDB->exec($sSQL, null, 'Database server failed to load '.$sModulePath.'/nominatim.so module');
-    }
 }

lib-sql/functions.sql

@@ -0,0 +1,20 @@
+{% include('functions/utils.sql') %}
+{% include('functions/normalization.sql') %}
+{% include('functions/ranking.sql') %}
+{% include('functions/importance.sql') %}
+{% include('functions/address_lookup.sql') %}
+{% include('functions/interpolation.sql') %}
+
+{% if 'place' in db.tables %}
+    {% include 'functions/place_triggers.sql' %}
+{% endif %}
+
+{% if 'placex' in db.tables %}
+    {% include 'functions/placex_triggers.sql' %}
+{% endif %}
+
+{% if 'location_postcode' in db.tables %}
+    {% include 'functions/postcode_triggers.sql' %}
+{% endif %}
+
+{% include('functions/partition-functions.sql') %}

lib-sql/functions/address_lookup.sql

@@ -121,7 +121,7 @@ BEGIN
   END IF;
 
   --then query tiger data
-  -- %NOTIGERDATA% IF 0 THEN
+  {% if config.get_bool('USE_US_TIGER_DATA') %}
   IF place IS NULL AND in_housenumber >= 0 THEN
     SELECT parent_place_id as place_id, 'us' as country_code,
            in_housenumber as housenumber, postcode,
@@ -133,9 +133,10 @@ BEGIN
       WHERE place_id = in_place_id
             AND in_housenumber between startnumber and endnumber;
   END IF;
-  -- %NOTIGERDATA% END IF;
+  {% endif %}
 
-  -- %NOAUXDATA% IF 0 THEN
+  -- then additional data
+  {% if config.get_bool('USE_AUX_LOCATION_DATA') %}
   IF place IS NULL THEN
     SELECT parent_place_id as place_id, 'us' as country_code,
            housenumber, postcode,
@@ -146,7 +147,7 @@ BEGIN
       FROM location_property_aux
       WHERE place_id = in_place_id;
   END IF;
-  -- %NOAUXDATA% END IF;
+  {% endif %}
 
   -- postcode table
   IF place IS NULL THEN

lib-sql/functions/normalization.sql

@@ -1,12 +1,12 @@
 -- Functions for term normalisation and access to the 'word' table.
 
 CREATE OR REPLACE FUNCTION transliteration(text) RETURNS text
-  AS '{modulepath}/nominatim.so', 'transliteration'
+  AS '{{ modulepath }}/nominatim.so', 'transliteration'
 LANGUAGE c IMMUTABLE STRICT;
 
 
 CREATE OR REPLACE FUNCTION gettokenstring(text) RETURNS text
-  AS '{modulepath}/nominatim.so', 'gettokenstring'
+  AS '{{ modulepath }}/nominatim.so', 'gettokenstring'
 LANGUAGE c IMMUTABLE STRICT;
 
 

lib-sql/functions/partition-functions.sql

@@ -37,13 +37,13 @@ DECLARE
   r nearfeaturecentr%rowtype;
 BEGIN
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
     FOR r IN
       SELECT place_id, keywords, rank_address, rank_search,
              min(ST_Distance(feature, centroid)) as distance,
              isguess, postcode, centroid
-      FROM location_area_large_-partition-
+      FROM location_area_large_{{ partition }}
       WHERE geometry && feature
         AND is_relevant_geometry(ST_Relate(geometry, feature), ST_GeometryType(feature))
         AND rank_address < maxrank
@@ -56,7 +56,7 @@ BEGIN
     END LOOP;
     RETURN;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
@@ -80,12 +80,12 @@ BEGIN
      CONTINUE;
    END IF;
 
--- start
+{% for partition in db.partitions %}
-    IF in_partition = -partition- THEN
+    IF in_partition = {{ partition }} THEN
         SELECT place_id, keywords, rank_address, rank_search,
                min(ST_Distance(feature, centroid)) as distance,
                isguess, postcode, centroid INTO r
-        FROM location_area_large_-partition-
+        FROM location_area_large_{{ partition }}
         WHERE geometry && ST_Expand(feature, item.extent)
           AND rank_address between item.from_rank and item.to_rank
           AND word_ids_from_name(item.name) && keywords
@@ -103,7 +103,7 @@ BEGIN
       END IF;
       CONTINUE;
     END IF;
--- end
+{% endfor %}
 
     RAISE EXCEPTION 'Unknown partition %', in_partition;
   END LOOP;
@@ -120,12 +120,12 @@ BEGIN
     RETURN TRUE;
   END IF;
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    DELETE from location_area_large_-partition- WHERE place_id = in_place_id;
+    DELETE from location_area_large_{{ partition }} WHERE place_id = in_place_id;
     RETURN TRUE;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 
@@ -150,13 +150,13 @@ BEGIN
     RETURN TRUE;
   END IF;
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    INSERT INTO location_area_large_-partition- (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry)
+    INSERT INTO location_area_large_{{ partition }} (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry)
       values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, postcode, in_centroid, in_geometry);
     RETURN TRUE;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
   RETURN FALSE;
@@ -173,9 +173,9 @@ DECLARE
   parent BIGINT;
 BEGIN
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    SELECT place_id FROM search_name_-partition-
+    SELECT place_id FROM search_name_{{ partition }}
       INTO parent
       WHERE name_vector && isin_token
             AND centroid && ST_Expand(point, 0.015)
@@ -183,7 +183,7 @@ BEGIN
       ORDER BY ST_Distance(centroid, point) ASC limit 1;
     RETURN parent;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
@@ -199,18 +199,18 @@ DECLARE
   parent BIGINT;
 BEGIN
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
     SELECT place_id
       INTO parent
-      FROM search_name_-partition-
+      FROM search_name_{{ partition }}
       WHERE name_vector && isin_token
             AND centroid && ST_Expand(point, 0.04)
             AND address_rank between 16 and 25
       ORDER BY ST_Distance(centroid, point) ASC limit 1;
     RETURN parent;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
@@ -223,16 +223,16 @@ create or replace function insertSearchName(
 RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    DELETE FROM search_name_-partition- values WHERE place_id = in_place_id;
+    DELETE FROM search_name_{{ partition }} values WHERE place_id = in_place_id;
     IF in_rank_address > 0 THEN
-      INSERT INTO search_name_-partition- (place_id, address_rank, name_vector, centroid)
+      INSERT INTO search_name_{{ partition }} (place_id, address_rank, name_vector, centroid)
         values (in_place_id, in_rank_address, in_name_vector, in_geometry);
     END IF;
     RETURN TRUE;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
   RETURN FALSE;
@@ -243,12 +243,12 @@ LANGUAGE plpgsql;
 create or replace function deleteSearchName(in_partition INTEGER, in_place_id BIGINT) RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    DELETE from search_name_-partition- WHERE place_id = in_place_id;
+    DELETE from search_name_{{ partition }} WHERE place_id = in_place_id;
     RETURN TRUE;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 
@@ -262,14 +262,14 @@ create or replace function insertLocationRoad(
 DECLARE
 BEGIN
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    DELETE FROM location_road_-partition- where place_id = in_place_id;
+    DELETE FROM location_road_{{ partition }} where place_id = in_place_id;
-    INSERT INTO location_road_-partition- (partition, place_id, country_code, geometry)
+    INSERT INTO location_road_{{ partition }} (partition, place_id, country_code, geometry)
       values (in_partition, in_place_id, in_country_code, in_geometry);
     RETURN TRUE;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
   RETURN FALSE;
@@ -281,12 +281,12 @@ create or replace function deleteRoad(in_partition INTEGER, in_place_id BIGINT)
 DECLARE
 BEGIN
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
-    DELETE FROM location_road_-partition- where place_id = in_place_id;
+    DELETE FROM location_road_{{ partition }} where place_id = in_place_id;
     RETURN TRUE;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 
@@ -303,12 +303,12 @@ DECLARE
   search_diameter FLOAT;
 BEGIN
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
     search_diameter := 0.00005;
     WHILE search_diameter < 0.1 LOOP
       FOR r IN
-        SELECT place_id FROM location_road_-partition-
+        SELECT place_id FROM location_road_{{ partition }}
           WHERE ST_DWithin(geometry, point, search_diameter)
           ORDER BY ST_Distance(geometry, point) ASC limit 1
       LOOP
@@ -318,7 +318,7 @@ BEGIN
     END LOOP;
     RETURN NULL;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
@@ -345,12 +345,12 @@ BEGIN
   p2 := ST_LineInterpolatePoint(line,0.5);
   p3 := ST_LineInterpolatePoint(line,1);
 
--- start
+{% for partition in db.partitions %}
-  IF in_partition = -partition- THEN
+  IF in_partition = {{ partition }} THEN
     search_diameter := 0.0005;
     WHILE search_diameter < 0.01 LOOP
       FOR r IN
-        SELECT place_id FROM location_road_-partition-
+        SELECT place_id FROM location_road_{{ partition }}
           WHERE ST_DWithin(line, geometry, search_diameter)
           ORDER BY (ST_distance(geometry, p1)+
                     ST_distance(geometry, p2)+
@@ -362,7 +362,7 @@ BEGIN
     END LOOP;
     RETURN NULL;
   END IF;
--- end
+{% endfor %}
 
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END

lib-sql/functions/place_triggers.sql

@@ -12,8 +12,10 @@ DECLARE
   partition INTEGER;
 BEGIN
 
-  --DEBUG: RAISE WARNING '-----------------------------------------------------------------------------------';
+  {% if debug %}
-  --DEBUG: RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
+    RAISE WARNING '-----------------------------------------------------------------------------------';
+    RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
+  {% endif %}
   -- filter wrong tupels
   IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN  
     INSERT INTO import_polygon_error (osm_type, osm_id, class, type, name, country_code, updated, errormessage, prevgeometry, newgeometry)
@@ -97,8 +99,8 @@ BEGIN
       DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
     END IF;
 
-    --DEBUG: RAISE WARNING 'Existing: %',existing.osm_id;
+    {% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
-    --DEBUG: RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id;
+    {% if debug %}RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id;{% endif %}
 
     -- Log and discard 
     IF existing.geometry is not null AND st_isvalid(existing.geometry) 
@@ -122,14 +124,20 @@ BEGIN
           (existingplacex.type != NEW.type)))
     THEN
 
+      {% if config.get_bool('LIMIT_REINDEXING') %}
       IF existingplacex.osm_type IS NOT NULL THEN
         -- sanity check: ignore admin_level changes on places with too many active children
         -- or we end up reindexing entire countries because somebody accidentally deleted admin_level
-        --LIMIT INDEXING: SELECT count(*) FROM (SELECT 'a' FROM placex , place_addressline where address_place_id = existingplacex.place_id and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub INTO i;
+        SELECT count(*) INTO i FROM
-        --LIMIT INDEXING: IF i > 100000 THEN
+          (SELECT 'a' FROM placex, place_addressline 
-        --LIMIT INDEXING:  RETURN null;
+            WHERE address_place_id = existingplacex.place_id
-        --LIMIT INDEXING: END IF;
+                  and placex.place_id = place_addressline.place_id
+                  and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub;
+        IF i > 100000 THEN
+          RETURN null;
+        END IF;
       END IF;
+      {% endif %}
 
       IF existing.osm_type IS NOT NULL THEN
         -- pathological case caused by the triggerless copy into place during initial import
@@ -144,7 +152,7 @@ BEGIN
         values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name,
                 NEW.admin_level, NEW.address, NEW.extratags, NEW.geometry);
 
-      --DEBUG: RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;
+      {% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
 
       RETURN NEW;
     END IF;
@@ -258,7 +266,7 @@ DECLARE
   has_rank BOOLEAN;
 BEGIN
 
-  --DEBUG: RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;
+  {% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
 
   -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
   IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN

lib-sql/functions/placex_triggers.sql

@@ -20,7 +20,7 @@ DECLARE
   location RECORD;
   parent RECORD;
 BEGIN
-    --DEBUG: RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;
+    {% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %}
 
     -- Is this object part of an associatedStreet relation?
     FOR location IN
@@ -58,7 +58,7 @@ BEGIN
                and poi_osm_id = any(x.nodes)
          LIMIT 1
       LOOP
-        --DEBUG: RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id;
+        {% if debug %}RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id;{% endif %}
         RETURN parent.parent_place_id;
       END LOOP;
 
@@ -71,11 +71,11 @@ BEGIN
                and p.geometry && bbox
                and w.id = p.osm_id and poi_osm_id = any(w.nodes)
       LOOP
-        --DEBUG: RAISE WARNING 'Node is part of way % ', location.osm_id;
+        {% if debug %}RAISE WARNING 'Node is part of way % ', location.osm_id;{% endif %}
 
         -- Way IS a road then we are on it - that must be our road
         IF location.rank_search < 28 THEN
-          --DEBUG: RAISE WARNING 'node in way that is a street %',location;
+          {% if debug %}RAISE WARNING 'node in way that is a street %',location;{% endif %}
           return location.place_id;
         END IF;
 
@@ -106,7 +106,7 @@ BEGIN
       ELSEIF ST_Area(bbox) < 0.005 THEN
         -- for smaller features get the nearest road
         SELECT getNearestRoadPlaceId(poi_partition, bbox) INTO parent_place_id;
-        --DEBUG: RAISE WARNING 'Checked for nearest way (%)', parent_place_id;
+        {% if debug %}RAISE WARNING 'Checked for nearest way (%)', parent_place_id;{% endif %}
       ELSE
         -- for larger features simply find the area with the largest rank that
         -- contains the bbox, only use addressable features
@@ -146,21 +146,21 @@ BEGIN
   IF bnd.osm_type = 'R' THEN
     -- see if we have any special relation members
     SELECT members FROM planet_osm_rels WHERE id = bnd.osm_id INTO relation_members;
-    --DEBUG: RAISE WARNING 'Got relation members';
+    {% if debug %}RAISE WARNING 'Got relation members';{% endif %}
 
     -- Search for relation members with role 'lable'.
     IF relation_members IS NOT NULL THEN
       FOR rel_member IN
         SELECT get_rel_node_members(relation_members, ARRAY['label']) as member
       LOOP
-        --DEBUG: RAISE WARNING 'Found label member %', rel_member.member;
+        {% if debug %}RAISE WARNING 'Found label member %', rel_member.member;{% endif %}
 
         FOR linked_placex IN
           SELECT * from placex
           WHERE osm_type = 'N' and osm_id = rel_member.member
             and class = 'place'
         LOOP
-          --DEBUG: RAISE WARNING 'Linked label member';
+          {% if debug %}RAISE WARNING 'Linked label member';{% endif %}
           RETURN linked_placex;
         END LOOP;
 
@@ -187,7 +187,7 @@ BEGIN
         AND placex.rank_search < 26 -- needed to select the right index
         AND _st_covers(bnd.geometry, placex.geometry)
     LOOP
-      --DEBUG: RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;
+      {% if debug %}RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;{% endif %}
       RETURN linked_placex;
     END LOOP;
   END IF;
@@ -203,14 +203,14 @@ BEGIN
         AND _st_covers(bnd.geometry, placex.geometry)
       ORDER BY make_standard_name(name->'name') = bnd_name desc
     LOOP
-      --DEBUG: RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;
+      {% if debug %}RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;{% endif %}
       RETURN linked_placex;
     END LOOP;
   END IF;
 
   -- Name searches can be done for ways as well as relations
   IF bnd_name is not null THEN
-    --DEBUG: RAISE WARNING 'Looking for nodes with matching names';
+    {% if debug %}RAISE WARNING 'Looking for nodes with matching names';{% endif %}
     FOR linked_placex IN
       SELECT placex.* from placex
       WHERE make_standard_name(name->'name') = bnd_name
@@ -225,7 +225,7 @@ BEGIN
         AND placex.rank_search < 26 -- needed to select the right index
         AND _st_covers(bnd.geometry, placex.geometry)
     LOOP
-      --DEBUG: RAISE WARNING 'Found matching place node %', linked_placex.osm_id;
+      {% if debug %}RAISE WARNING 'Found matching place node %', linked_placex.osm_id;{% endif %}
       RETURN linked_placex;
     END LOOP;
   END IF;
@@ -285,10 +285,10 @@ BEGIN
                                                    address, country)
     ORDER BY rank_address, distance, isguess desc
   LOOP
-    IF NOT %REVERSE-ONLY% THEN
+    {% if not db.reverse_only %}
       nameaddress_vector := array_merge(nameaddress_vector,
                                         location.keywords::int[]);
-    END IF;
+    {% endif %}
 
     IF location.place_id is not null THEN
       location_isaddress := not address_havelevel[location.rank_address];
@@ -362,10 +362,10 @@ BEGIN
     END IF;
 
     -- Add it to the list of search terms
-    IF NOT %REVERSE-ONLY% THEN
+    {% if not db.reverse_only %}
       nameaddress_vector := array_merge(nameaddress_vector,
                                         location.keywords::integer[]);
-    END IF;
+    {% endif %}
 
     INSERT INTO place_addressline (place_id, address_place_id, fromarea,
                                      isaddress, distance, cached_rank_address)
@@ -388,7 +388,7 @@ DECLARE
   diameter FLOAT;
   classtable TEXT;
 BEGIN
-  --DEBUG: RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
+  {% if debug %}RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;{% endif %}
 
   NEW.place_id := nextval('seq_place');
   NEW.indexed_status := 1; --STATUS_NEW
@@ -440,9 +440,10 @@ BEGIN
 
   END IF;
 
-  --DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
+  {% if debug %}RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;{% endif %}
 
-  RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
+{% if not disable_diff_updates %}
+  -- The following is not needed until doing diff updates, and slows the main index process down
 
   IF NEW.osm_type = 'N' and NEW.rank_search > 28 THEN
       -- might be part of an interpolation
@@ -497,6 +498,8 @@ BEGIN
     USING NEW.place_id, ST_Centroid(NEW.geometry);
   END IF;
 
+{% endif %} -- not disable_diff_updates
+
   RETURN NEW;
 
 END;
@@ -534,7 +537,7 @@ DECLARE
 BEGIN
   -- deferred delete
   IF OLD.indexed_status = 100 THEN
-    --DEBUG: RAISE WARNING 'placex_update delete % %',NEW.osm_type,NEW.osm_id;
+    {% if debug %}RAISE WARNING 'placex_update delete % %',NEW.osm_type,NEW.osm_id;{% endif %}
     delete from placex where place_id = OLD.place_id;
     RETURN NULL;
   END IF;
@@ -543,13 +546,13 @@ BEGIN
     RETURN NEW;
   END IF;
 
-  --DEBUG: RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id;
+  {% if debug %}RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id;{% endif %}
 
   NEW.indexed_date = now();
 
-  IF NOT %REVERSE-ONLY% THEN
+  {% if 'search_name' in db.tables %}
     DELETE from search_name WHERE place_id = NEW.place_id;
-  END IF;
+  {% endif %}
   result := deleteSearchName(NEW.partition, NEW.place_id);
   DELETE FROM place_addressline WHERE place_id = NEW.place_id;
   result := deleteRoad(NEW.partition, NEW.place_id);
@@ -562,7 +565,7 @@ BEGIN
   NEW.address := NEW.address - '_unlisted_place'::TEXT;
 
   IF NEW.linked_place_id is not null THEN
-    --DEBUG: RAISE WARNING 'place already linked to %', NEW.linked_place_id;
+    {% if debug %}RAISE WARNING 'place already linked to %', NEW.linked_place_id;{% endif %}
     RETURN NEW;
   END IF;
 
@@ -578,7 +581,7 @@ BEGIN
   -- Speed up searches - just use the centroid of the feature
   -- cheaper but less acurate
   NEW.centroid := ST_PointOnSurface(NEW.geometry);
-  --DEBUG: RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid);
+  {% if debug %}RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid);{% endif %}
 
   -- recompute the ranks, they might change when linking changes
   SELECT * INTO NEW.rank_search, NEW.rank_address
@@ -658,7 +661,7 @@ BEGIN
     parent_address_level := 3;
   END IF;
 
-  --DEBUG: RAISE WARNING 'Copy over address tags';
+  {% if debug %}RAISE WARNING 'Copy over address tags';{% endif %}
   -- housenumber is a computed field, so start with an empty value
   NEW.housenumber := NULL;
   IF NEW.address is not NULL THEN
@@ -707,7 +710,7 @@ BEGIN
     END IF;
     NEW.partition := get_partition(NEW.country_code);
   END IF;
-  --DEBUG: RAISE WARNING 'Country updated: "%"', NEW.country_code;
+  {% if debug %}RAISE WARNING 'Country updated: "%"', NEW.country_code;{% endif %}
 
   -- waterway ways are linked when they are part of a relation and have the same class/type
   IF NEW.osm_type = 'R' and NEW.class = 'waterway' THEN
@@ -715,21 +718,21 @@ BEGIN
       LOOP
           FOR i IN 1..array_upper(relation_members, 1) BY 2 LOOP
               IF relation_members[i+1] in ('', 'main_stream', 'side_stream') AND substring(relation_members[i],1,1) = 'w' THEN
-                --DEBUG: RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i];
+                {% if debug %}RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i];{% endif %}
                 FOR linked_node_id IN SELECT place_id FROM placex
                   WHERE osm_type = 'W' and osm_id = substring(relation_members[i],2,200)::bigint
                   and class = NEW.class and type in ('river', 'stream', 'canal', 'drain', 'ditch')
                   and ( relation_members[i+1] != 'side_stream' or NEW.name->'name' = name->'name')
                 LOOP
                   UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = linked_node_id;
-                  IF NOT %REVERSE-ONLY% THEN
+                  {% if 'search_name' in db.tables %}
                     DELETE FROM search_name WHERE place_id = linked_node_id;
-                  END IF;
+                  {% endif %}
                 END LOOP;
               END IF;
           END LOOP;
       END LOOP;
-      --DEBUG: RAISE WARNING 'Waterway processed';
+      {% if debug %}RAISE WARNING 'Waterway processed';{% endif %}
   END IF;
 
   NEW.importance := null;
@@ -737,13 +740,13 @@ BEGIN
     FROM compute_importance(NEW.extratags, NEW.country_code, NEW.osm_type, NEW.osm_id)
     INTO NEW.wikipedia,NEW.importance;
 
---DEBUG: RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;
+{% if debug %}RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;{% endif %}
 
   -- ---------------------------------------------------------------------------
   -- For low level elements we inherit from our parent road
   IF NEW.rank_search > 27 THEN
 
-    --DEBUG: RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;
+    {% if debug %}RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;{% endif %}
     NEW.parent_place_id := null;
 
     -- if we have a POI and there is no address information,
@@ -791,7 +794,7 @@ BEGIN
       END IF;
 
       NEW.country_code := location.country_code;
-      --DEBUG: RAISE WARNING 'Got parent details from search name';
+      {% if debug %}RAISE WARNING 'Got parent details from search name';{% endif %}
 
       -- determine postcode
       IF NEW.address is not null AND NEW.address ? 'postcode' THEN
@@ -812,13 +815,14 @@ BEGIN
                                    name_vector, NEW.rank_search, NEW.rank_address,
                                    upper(trim(NEW.address->'postcode')), NEW.geometry,
                                    NEW.centroid);
-            --DEBUG: RAISE WARNING 'Place added to location table';
+            {% if debug %}RAISE WARNING 'Place added to location table';{% endif %}
           END IF;
 
       END IF;
 
-      IF not %REVERSE-ONLY% AND (array_length(name_vector, 1) is not NULL
+      {% if not db.reverse_only %}
-         OR inherited_address is not NULL OR NEW.address is not NULL)
+      IF array_length(name_vector, 1) is not NULL
+         OR inherited_address is not NULL OR NEW.address is not NULL
       THEN
         SELECT * INTO name_vector, nameaddress_vector
           FROM create_poi_search_terms(NEW.place_id,
@@ -834,9 +838,10 @@ BEGIN
                  VALUES (NEW.place_id, NEW.rank_search, NEW.rank_address,
                          NEW.importance, NEW.country_code, name_vector,
                          nameaddress_vector, NEW.centroid);
-          --DEBUG: RAISE WARNING 'Place added to search table';
+          {% if debug %}RAISE WARNING 'Place added to search table';{% endif %}
         END IF;
       END IF;
+      {% endif %}
 
       RETURN NEW;
     END IF;
@@ -845,10 +850,10 @@ BEGIN
 
   -- ---------------------------------------------------------------------------
   -- Full indexing
-  --DEBUG: RAISE WARNING 'Using full index mode for % %', NEW.osm_type, NEW.osm_id;
+  {% if debug %}RAISE WARNING 'Using full index mode for % %', NEW.osm_type, NEW.osm_id;{% endif %}
   SELECT * INTO location FROM find_linked_place(NEW);
   IF location.place_id is not null THEN
-    --DEBUG: RAISE WARNING 'Linked %', location;
+    {% if debug %}RAISE WARNING 'Linked %', location;{% endif %}
 
     -- Use the linked point as the centre point of the geometry,
     -- but only if it is within the area of the boundary.
@@ -857,7 +862,7 @@ BEGIN
         NEW.centroid := geom;
     END IF;
 
-    --DEBUG: RAISE WARNING 'parent address: % rank address: %', parent_address_level, location.rank_address;
+    {% if debug %}RAISE WARNING 'parent address: % rank address: %', parent_address_level, location.rank_address;{% endif %}
     IF location.rank_address > parent_address_level
        and location.rank_address < 26
     THEN
@@ -878,9 +883,9 @@ BEGIN
     UPDATE placex set linked_place_id = NEW.place_id
       WHERE place_id = location.place_id;
     -- ensure that those places are not found anymore
-    IF NOT %REVERSE-ONLY% THEN
+    {% if 'search_name' in db.tables %}
       DELETE FROM search_name WHERE place_id = location.place_id;
-    END IF;
+    {% endif %}
     PERFORM deleteLocationArea(NEW.partition, location.place_id, NEW.rank_search);
 
     SELECT wikipedia, importance
@@ -918,7 +923,7 @@ BEGIN
      AND NEW.country_code IS NOT NULL AND NEW.osm_type = 'R'
   THEN
     PERFORM create_country(NEW.name, lower(NEW.country_code));
-    --DEBUG: RAISE WARNING 'Country names updated';
+    {% if debug %}RAISE WARNING 'Country names updated';{% endif %}
 
     -- Also update the list of country names. Adding an additional sanity
     -- check here: make sure the country does overlap with the area where
@@ -928,7 +933,7 @@ BEGIN
        WHERE ST_Covers(geometry, NEW.centroid) and country_code = NEW.country_code
        LIMIT 1
     LOOP
-      --DEBUG: RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name;
+      {% if debug %}RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name;{% endif %}
       UPDATE country_name SET name = name || NEW.name WHERE country_code = NEW.country_code;
     END LOOP;
   END IF;
@@ -960,7 +965,7 @@ BEGIN
                                     NEW.address, geom, NEW.country_code)
     INTO NEW.parent_place_id, NEW.postcode, nameaddress_vector;
 
-  --DEBUG: RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector;
+  {% if debug %}RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector;{% endif %}
 
   IF NEW.address is not null AND NEW.address ? 'postcode' 
      AND NEW.address->'postcode' not similar to '%(,|;)%' THEN
@@ -976,30 +981,30 @@ BEGIN
 
     IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
       result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry, NEW.centroid);
-      --DEBUG: RAISE WARNING 'added to location (full)';
+      {% if debug %}RAISE WARNING 'added to location (full)';{% endif %}
     END IF;
 
     IF NEW.rank_search between 26 and 27 and NEW.class = 'highway' THEN
       result := insertLocationRoad(NEW.partition, NEW.place_id, NEW.country_code, NEW.geometry);
-      --DEBUG: RAISE WARNING 'insert into road location table (full)';
+      {% if debug %}RAISE WARNING 'insert into road location table (full)';{% endif %}
     END IF;
 
     result := insertSearchName(NEW.partition, NEW.place_id, name_vector,
                                NEW.rank_search, NEW.rank_address, NEW.geometry);
-    --DEBUG: RAISE WARNING 'added to search name (full)';
+    {% if debug %}RAISE WARNING 'added to search name (full)';{% endif %}
 
-    IF NOT %REVERSE-ONLY% THEN
+    {% if not db.reverse_only %}
         INSERT INTO search_name (place_id, search_rank, address_rank,
                                  importance, country_code, name_vector,
                                  nameaddress_vector, centroid)
                VALUES (NEW.place_id, NEW.rank_search, NEW.rank_address,
                        NEW.importance, NEW.country_code, name_vector,
                        nameaddress_vector, NEW.centroid);
-    END IF;
+    {% endif %}
 
   END IF;
 
-  --DEBUG: RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id;
+  {% if debug %}RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id;{% endif %}
 
   RETURN NEW;
 END;
@@ -1018,9 +1023,9 @@ BEGIN
 
   IF OLD.linked_place_id is null THEN
     update placex set linked_place_id = null, indexed_status = 2 where linked_place_id = OLD.place_id and indexed_status = 0;
-    --DEBUG: RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id;{% endif %}
     update placex set linked_place_id = null where linked_place_id = OLD.place_id;
-    --DEBUG: RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id;{% endif %}
   ELSE
     update placex set indexed_status = 2 where place_id = OLD.linked_place_id and indexed_status = 0;
   END IF;
@@ -1028,44 +1033,44 @@ BEGIN
   IF OLD.rank_address < 30 THEN
 
     -- mark everything linked to this place for re-indexing
-    --DEBUG: RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id;{% endif %}
     UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = OLD.place_id 
       and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress;
 
-    --DEBUG: RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id;{% endif %}
     DELETE FROM place_addressline where address_place_id = OLD.place_id;
 
-    --DEBUG: RAISE WARNING 'placex_delete:05 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:05 % %',OLD.osm_type,OLD.osm_id;{% endif %}
     b := deleteRoad(OLD.partition, OLD.place_id);
 
-    --DEBUG: RAISE WARNING 'placex_delete:06 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:06 % %',OLD.osm_type,OLD.osm_id;{% endif %}
     update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0;
-    --DEBUG: RAISE WARNING 'placex_delete:07 % %',OLD.osm_type,OLD.osm_id;
+    {% if debug %}RAISE WARNING 'placex_delete:07 % %',OLD.osm_type,OLD.osm_id;{% endif %}
     -- reparenting also for OSM Interpolation Lines (and for Tiger?)
     update location_property_osmline set indexed_status = 2 where indexed_status = 0 and parent_place_id = OLD.place_id;
 
   END IF;
 
-  --DEBUG: RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;
+  {% if debug %}RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;{% endif %}
 
   IF OLD.rank_address < 26 THEN
     b := deleteLocationArea(OLD.partition, OLD.place_id, OLD.rank_search);
   END IF;
 
-  --DEBUG: RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;
+  {% if debug %}RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;{% endif %}
 
   IF OLD.name is not null THEN
-    IF NOT %REVERSE-ONLY% THEN
+    {% if 'search_name' in db.tables %}
       DELETE from search_name WHERE place_id = OLD.place_id;
-    END IF;
+    {% endif %}
     b := deleteSearchName(OLD.partition, OLD.place_id);
   END IF;
 
-  --DEBUG: RAISE WARNING 'placex_delete:10 % %',OLD.osm_type,OLD.osm_id;
+  {% if debug %}RAISE WARNING 'placex_delete:10 % %',OLD.osm_type,OLD.osm_id;{% endif %}
 
   DELETE FROM place_addressline where place_id = OLD.place_id;
 
-  --DEBUG: RAISE WARNING 'placex_delete:11 % %',OLD.osm_type,OLD.osm_id;
+  {% if debug %}RAISE WARNING 'placex_delete:11 % %',OLD.osm_type,OLD.osm_id;{% endif %}
 
   -- remove from tables for special search
   classtable := 'place_classtype_' || OLD.class || '_' || OLD.type;
@@ -1074,7 +1079,7 @@ BEGIN
     EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id;
   END IF;
 
-  --DEBUG: RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id;
+  {% if debug %}RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id;{% endif %}
 
   RETURN OLD;
 

lib-sql/functions/utils.sql

@@ -237,7 +237,7 @@ BEGIN
     IF word_ids is not null THEN
       parent_place_id := getNearestNamedRoadPlaceId(partition, centroid, word_ids);
       IF parent_place_id is not null THEN
-        --DEBUG: RAISE WARNING 'Get parent form addr:street: %', parent_place_id;
+        {% if debug %}RAISE WARNING 'Get parent form addr:street: %', parent_place_id;{% endif %}
         RETURN parent_place_id;
       END IF;
     END IF;
@@ -249,7 +249,7 @@ BEGIN
     IF word_ids is not null THEN
       parent_place_id := getNearestNamedPlacePlaceId(partition, centroid, word_ids);
       IF parent_place_id is not null THEN
-        --DEBUG: RAISE WARNING 'Get parent form addr:place: %', parent_place_id;
+        {% if debug %}RAISE WARNING 'Get parent form addr:place: %', parent_place_id;{% endif %}
         RETURN parent_place_id;
       END IF;
     END IF;

lib-sql/indices.sql

@@ -0,0 +1,67 @@
+-- Indices used only during search and update.
+-- These indices are created only after the indexing process is done.
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_word_word_id
+  ON word USING BTREE (word_id) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_place_addressline_address_place_id
+  ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_search
+  ON placex USING BTREE (rank_search) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_address
+  ON placex USING BTREE (rank_address) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_parent_place_id
+  ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}}
+  WHERE parent_place_id IS NOT NULL;
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_lookupPolygon
+  ON placex USING gist (geometry) {{db.tablespace.search_index}}
+  WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
+    AND rank_address between 4 and 25 AND type != 'postcode'
+    AND name is not null AND indexed_status = 0 AND linked_place_id is null;
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_placeNode
+  ON placex USING gist (geometry) {{db.tablespace.search_index}}
+  WHERE osm_type = 'N' AND rank_search between 5 and 25
+    AND class = 'place' AND type != 'postcode'
+    AND name is not null AND indexed_status = 0 AND linked_place_id is null;
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_place_id
+  ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_osm_id
+  ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}};
+
+CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_postcode_id
+  ON location_postcode USING BTREE (place_id) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_postcode_postcode
+  ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}};
+
+-- Indices only needed for updating.
+
+{% if not drop %}
+  CREATE INDEX {{sql.if_index_not_exists}} idx_placex_pendingsector
+    ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}}
+    WHERE indexed_status > 0;
+
+  CREATE INDEX {{sql.if_index_not_exists}} idx_location_area_country_place_id
+    ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}};
+
+  CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_place_osm_unique
+    ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}};
+{% endif %}
+
+-- Indices only needed for search.
+
+{% if 'search_name' in db.tables %}
+  CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_nameaddress_vector
+    ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
+  CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_name_vector
+    ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
+  CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_centroid
+    ON search_name USING GIST (centroid) {{db.tablespace.search_index}};
+{% endif %}

lib-sql/indices.src.sql

@@ -1,30 +0,0 @@
--- Indices used only during search and update.
--- These indices are created only after the indexing process is done.
-
-CREATE INDEX idx_word_word_id on word USING BTREE (word_id) {ts:search-index};
-
-CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index};
-
-DROP INDEX IF EXISTS idx_placex_rank_search;
-CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index};
-CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index};
-CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL;
-
-CREATE INDEX idx_placex_geometry_reverse_lookupPolygon
-  ON placex USING gist (geometry) {ts:search-index}
-  WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
-    AND rank_address between 4 and 25 AND type != 'postcode'
-    AND name is not null AND indexed_status = 0 AND linked_place_id is null;
-CREATE INDEX idx_placex_geometry_reverse_placeNode
-  ON placex USING gist (geometry) {ts:search-index}
-  WHERE osm_type = 'N' AND rank_search between 5 and 25
-    AND class = 'place' AND type != 'postcode'
-    AND name is not null AND indexed_status = 0 AND linked_place_id is null;
-
-GRANT SELECT ON table country_osm_grid to "{www-user}";
-
-CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
-CREATE INDEX idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index};
-
-CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
-CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};

lib-sql/indices_search.src.sql

@@ -1,6 +0,0 @@
--- Indices used for /search API.
--- These indices are created only after the indexing process is done.
-
-CREATE INDEX idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index};
-CREATE INDEX idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index};
-CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index};

lib-sql/indices_updates.src.sql

@@ -1,9 +0,0 @@
--- Indices used only during search and update.
--- These indices are created only after the indexing process is done.
-
-CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_address,geometry_sector) {ts:address-index} where indexed_status > 0;
-
-CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index};
-
-DROP INDEX CONCURRENTLY IF EXISTS place_id_idx;
-CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};

lib-sql/partition-tables.src.sql

@@ -7,24 +7,24 @@ CREATE TABLE search_name_blank (
   );
 
 
--- start
+{% for partition in db.partitions %}
-CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large) {ts:address-data};
+  CREATE TABLE location_area_large_{{ partition }} () INHERITS (location_area_large) {{db.tablespace.address_data}};
-CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id) {ts:address-index};
+  CREATE INDEX idx_location_area_large_{{ partition }}_place_id ON location_area_large_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry) {ts:address-index};
+  CREATE INDEX idx_location_area_large_{{ partition }}_geometry ON location_area_large_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
 
-CREATE TABLE search_name_-partition- () INHERITS (search_name_blank) {ts:address-data};
+  CREATE TABLE search_name_{{ partition }} () INHERITS (search_name_blank) {{db.tablespace.address_data}};
-CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id) {ts:address-index};
+  CREATE INDEX idx_search_name_{{ partition }}_place_id ON search_name_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-CREATE INDEX idx_search_name_-partition-_centroid_street ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 26 and 27;
+  CREATE INDEX idx_search_name_{{ partition }}_centroid_street ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 26 and 27;
-CREATE INDEX idx_search_name_-partition-_centroid_place ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 2 and 25;
+  CREATE INDEX idx_search_name_{{ partition }}_centroid_place ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 2 and 25;
 
-DROP TABLE IF EXISTS location_road_-partition-;
+  DROP TABLE IF EXISTS location_road_{{ partition }};
-CREATE TABLE location_road_-partition- (
+  CREATE TABLE location_road_{{ partition }} (
-  place_id BIGINT,
+    place_id BIGINT,
-  partition SMALLINT,
+    partition SMALLINT,
-  country_code VARCHAR(2),
+    country_code VARCHAR(2),
-  geometry GEOMETRY(Geometry, 4326)
+    geometry GEOMETRY(Geometry, 4326)
-  ) {ts:address-data};
+    ) {{db.tablespace.address_data}};
-CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry) {ts:address-index};
+  CREATE INDEX idx_location_road_{{ partition }}_geometry ON location_road_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
-CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id) {ts:address-index};
+  CREATE INDEX idx_location_road_{{ partition }}_place_id ON location_road_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
 
--- end
+{% endfor %}

lib-sql/tables.sql

@@ -4,7 +4,7 @@ CREATE TABLE import_status (
   sequence_id integer,
   indexed boolean
   );
-GRANT SELECT ON import_status TO "{www-user}" ;
+GRANT SELECT ON import_status TO "{{config.DATABASE_WEBUSER}}" ;
 
 drop table if exists import_osmosis_log;
 CREATE TABLE import_osmosis_log (
@@ -30,18 +30,18 @@ CREATE TABLE new_query_log (
   secret text
   );
 CREATE INDEX idx_new_query_log_starttime ON new_query_log USING BTREE (starttime);
-GRANT INSERT ON new_query_log TO "{www-user}" ;
+GRANT INSERT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
-GRANT UPDATE ON new_query_log TO "{www-user}" ;
+GRANT UPDATE ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT ON new_query_log TO "{www-user}" ;
+GRANT SELECT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
 
-GRANT SELECT ON TABLE country_name TO "{www-user}";
+GRANT SELECT ON TABLE country_name TO "{{config.DATABASE_WEBUSER}}";
 
 DROP TABLE IF EXISTS nominatim_properties;
 CREATE TABLE nominatim_properties (
     property TEXT,
     value TEXT
 );
-GRANT SELECT ON TABLE nominatim_properties TO "{www-user}";
+GRANT SELECT ON TABLE nominatim_properties TO "{{config.DATABASE_WEBUSER}}";
 
 drop table IF EXISTS word;
 CREATE TABLE word (
@@ -53,9 +53,9 @@ CREATE TABLE word (
   country_code varchar(2),
   search_name_count INTEGER,
   operator TEXT
-  ) {ts:search-data};
+  ) {{db.tablespace.search_data}};
-CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {ts:search-index};
+CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {{db.tablespace.search_index}};
-GRANT SELECT ON word TO "{www-user}" ;
+GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}" ;
 DROP SEQUENCE IF EXISTS seq_word;
 CREATE SEQUENCE seq_word start 1;
 
@@ -80,8 +80,8 @@ CREATE TABLE location_area_country (
   place_id BIGINT,
   country_code varchar(2),
   geometry GEOMETRY(Geometry, 4326)
-  ) {ts:address-data};
+  ) {{db.tablespace.address_data}};
-CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {ts:address-index};
+CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {{db.tablespace.address_index}};
 
 
 drop table IF EXISTS location_property CASCADE;
@@ -98,7 +98,7 @@ CREATE TABLE location_property_aux () INHERITS (location_property);
 CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
 CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
 CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
-GRANT SELECT ON location_property_aux TO "{www-user}";
+GRANT SELECT ON location_property_aux TO "{{config.DATABASE_WEBUSER}}";
 
 CREATE TABLE location_property_tiger (
   place_id BIGINT,
@@ -109,7 +109,7 @@ CREATE TABLE location_property_tiger (
   linegeo GEOMETRY,
   interpolationtype TEXT,
   postcode TEXT);
-GRANT SELECT ON location_property_tiger TO "{www-user}";
+GRANT SELECT ON location_property_tiger TO "{{config.DATABASE_WEBUSER}}";
 
 drop table if exists location_property_osmline;
 CREATE TABLE location_property_osmline (
@@ -127,13 +127,14 @@ CREATE TABLE location_property_osmline (
     address HSTORE,
     postcode TEXT,
     country_code VARCHAR(2)
-  ){ts:search-data};
+  ){{db.tablespace.search_data}};
-CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {ts:search-index};
+CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {{db.tablespace.search_index}};
-CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {ts:address-index};
+CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {{db.tablespace.address_index}};
-CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {ts:search-index};
+CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {{db.tablespace.search_index}};
-GRANT SELECT ON location_property_osmline TO "{www-user}";
+GRANT SELECT ON location_property_osmline TO "{{config.DATABASE_WEBUSER}}";
 
 drop table IF EXISTS search_name;
+{% if not db.reverse_only %}
 CREATE TABLE search_name (
   place_id BIGINT,
   importance FLOAT,
@@ -143,8 +144,10 @@ CREATE TABLE search_name (
   nameaddress_vector integer[],
   country_code varchar(2),
   centroid GEOMETRY(Geometry, 4326)
-  ) {ts:search-data};
+  ) {{db.tablespace.search_data}};
-CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {ts:search-index};
+CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {{db.tablespace.search_index}};
+GRANT SELECT ON search_name to "{{config.DATABASE_WEBUSER}}" ;
+{% endif %}
 
 drop table IF EXISTS place_addressline;
 CREATE TABLE place_addressline (
@@ -154,8 +157,8 @@ CREATE TABLE place_addressline (
   cached_rank_address SMALLINT,
   fromarea boolean,
   isaddress boolean
-  ) {ts:search-data};
+  ) {{db.tablespace.search_data}};
-CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {ts:search-index};
+CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}};
 
 drop table if exists placex;
 CREATE TABLE placex (
@@ -175,24 +178,23 @@ CREATE TABLE placex (
   housenumber TEXT,
   postcode TEXT,
   centroid GEOMETRY(Geometry, 4326)
-  ) {ts:search-data};
+  ) {{db.tablespace.search_data}};
-CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {ts:search-index};
+CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
-CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {ts:search-index};
+CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tablespace.search_index}};
-CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {ts:address-index} WHERE linked_place_id IS NOT NULL;
+CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL;
-CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {ts:address-index};
+CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}};
-CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {ts:search-index};
+CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}};
-CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {ts:address-index} WHERE osm_type='N' and rank_search < 26;
+CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {{db.tablespace.address_index}} WHERE osm_type='N' and rank_search < 26;
-CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {ts:address-index} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
+CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
 
 DROP SEQUENCE IF EXISTS seq_place;
 CREATE SEQUENCE seq_place start 1;
-GRANT SELECT on placex to "{www-user}" ;
+GRANT SELECT on placex to "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT ON search_name to "{www-user}" ;
+GRANT SELECT on place_addressline to "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT on place_addressline to "{www-user}" ;
+GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT ON seq_word to "{www-user}" ;
+GRANT SELECT ON planet_osm_ways to "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT ON planet_osm_ways to "{www-user}" ;
+GRANT SELECT ON planet_osm_rels to "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT ON planet_osm_rels to "{www-user}" ;
+GRANT SELECT on location_area to "{{config.DATABASE_WEBUSER}}" ;
-GRANT SELECT on location_area to "{www-user}" ;
 
 -- Table for synthetic postcodes.
 DROP TABLE IF EXISTS location_postcode;
@@ -207,8 +209,8 @@ CREATE TABLE location_postcode (
   postcode TEXT,
   geometry GEOMETRY(Geometry, 4326)
   );
-CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {ts:address-index};
+CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {{db.tablespace.address_index}};
-GRANT SELECT ON location_postcode TO "{www-user}" ;
+GRANT SELECT ON location_postcode TO "{{config.DATABASE_WEBUSER}}" ;
 
 DROP TABLE IF EXISTS import_polygon_error;
 CREATE TABLE import_polygon_error (
@@ -224,7 +226,7 @@ CREATE TABLE import_polygon_error (
   newgeometry GEOMETRY(Geometry, 4326)
   );
 CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
-GRANT SELECT ON import_polygon_error TO "{www-user}";
+GRANT SELECT ON import_polygon_error TO "{{config.DATABASE_WEBUSER}}";
 
 DROP TABLE IF EXISTS import_polygon_delete;
 CREATE TABLE import_polygon_delete (
@@ -234,7 +236,7 @@ CREATE TABLE import_polygon_delete (
   type TEXT NOT NULL
   );
 CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id);
-GRANT SELECT ON import_polygon_delete TO "{www-user}";
+GRANT SELECT ON import_polygon_delete TO "{{config.DATABASE_WEBUSER}}";
 
 DROP SEQUENCE IF EXISTS file;
 CREATE SEQUENCE file start 1;
@@ -268,3 +270,5 @@ ALTER TABLE ONLY wikipedia_redirect ADD CONSTRAINT wikipedia_redirect_pkey PRIMA
 -- osm2pgsql does not create indexes on the middle tables for Nominatim
 -- Add one for lookup of associated street relations.
 CREATE INDEX planet_osm_rels_parts_associated_idx ON planet_osm_rels USING gin(parts) WHERE tags @> ARRAY['associatedStreet'];
+
+GRANT SELECT ON table country_osm_grid to "{{config.DATABASE_WEBUSER}}";

nominatim/clicmd/setup.py

@@ -79,20 +79,22 @@ class SetupAll:
                                             drop=args.no_updates,
                                             ignore_errors=args.ignore_errors)
 
-            LOG.warning('Create functions (1st pass)')
             with connect(args.config.get_libpq_dsn()) as conn:
+                LOG.warning('Create functions (1st pass)')
                 refresh.create_functions(conn, args.config, args.sqllib_dir,
                                          False, False)
-
+                LOG.warning('Create tables')
-            LOG.warning('Create tables')
+                database_import.create_tables(conn, args.config, args.sqllib_dir,
-            params = ['setup.php', '--create-tables', '--create-partition-tables']
+                                              reverse_only=args.reverse_only)
-            if args.reverse_only:
+                refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
-                params.append('--reverse-only')
+                LOG.warning('Create functions (2nd pass)')
-            run_legacy_script(*params, nominatim_env=args,
+                refresh.create_functions(conn, args.config, args.sqllib_dir,
-                              throw_on_fail=not args.ignore_errors)
+                                         False, False)
-
+                LOG.warning('Create table triggers')
-            LOG.warning('Create functions (2nd pass)')
+                database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
-            with connect(args.config.get_libpq_dsn()) as conn:
+                LOG.warning('Create partition tables')
+                database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
+                LOG.warning('Create functions (3rd pass)')
                 refresh.create_functions(conn, args.config, args.sqllib_dir,
                                          False, False)
 
@@ -124,10 +126,12 @@ class SetupAll:
             indexer.index_full(analyse=not args.index_noanalyse)
 
         LOG.warning('Post-process tables')
-        params = ['setup.php', '--create-search-indices', '--create-country-names']
+        with connect(args.config.get_libpq_dsn()) as conn:
-        if args.no_updates:
+            database_import.create_search_indices(conn, args.config,
-            params.append('--drop')
+                                                  args.sqllib_dir,
-        run_legacy_script(*params, nominatim_env=args, throw_on_fail=not args.ignore_errors)
+                                                  drop=args.no_updates)
+        run_legacy_script('setup.php', '--create-country-names',
+                          nominatim_env=args, throw_on_fail=not args.ignore_errors)
 
         webdir = args.project_dir / 'website'
         LOG.warning('Setup website at %s', webdir)

nominatim/clicmd/transition.py

@@ -35,8 +35,14 @@ class AdminTransition:
                            help='Import a osm file')
         group.add_argument('--load-data', action='store_true',
                            help='Copy data to live tables from import table')
+        group.add_argument('--create-tables', action='store_true',
+                           help='Create main tables')
+        group.add_argument('--create-partition-tables', action='store_true',
+                           help='Create required partition tables')
         group.add_argument('--index', action='store_true',
                            help='Index the data')
+        group.add_argument('--create-search-indices', action='store_true',
+                           help='Create additional indices required for search and update')
         group = parser.add_argument_group('Options')
         group.add_argument('--no-partitions', action='store_true',
                            help='Do not partition search indices')
@@ -50,10 +56,13 @@ class AdminTransition:
                            help='Do not perform analyse operations during index')
         group.add_argument('--ignore-errors', action='store_true',
                            help="Ignore certain erros on import.")
+        group.add_argument('--reverse-only', action='store_true',
+                           help='Do not create search tables and indexes')
 
     @staticmethod
     def run(args):
         from ..tools import database_import
+        from ..tools import refresh
 
         if args.create_db:
             LOG.warning('Create DB')
@@ -80,6 +89,20 @@ class AdminTransition:
                                             drop=args.drop,
                                             ignore_errors=args.ignore_errors)
 
+        if args.create_tables:
+            LOG.warning('Create Tables')
+            with connect(args.config.get_libpq_dsn()) as conn:
+                database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only)
+                refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
+                refresh.create_functions(conn, args.config, args.sqllib_dir,
+                                         enable_diff_updates=False)
+                database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
+
+        if args.create_partition_tables:
+            LOG.warning('Create Partition Tables')
+            with connect(args.config.get_libpq_dsn()) as conn:
+                database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
+
         if args.load_data:
             LOG.warning('Load data')
             with connect(args.config.get_libpq_dsn()) as conn:
@@ -99,3 +122,8 @@ class AdminTransition:
             from ..indexer.indexer import Indexer
             indexer = Indexer(args.config.get_libpq_dsn(), args.threads or 1)
             indexer.index_full()
+
+        if args.create_search_indices:
+            LOG.warning('Create Search indices')
+            with connect(args.config.get_libpq_dsn()) as conn:
+                database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop)

nominatim/db/sql_preprocessor.py

@@ -0,0 +1,94 @@
+"""
+Preprocessing of SQL files.
+"""
+import jinja2
+
+
+def _get_partitions(conn):
+    """ Get the set of partitions currently in use.
+    """
+    with conn.cursor() as cur:
+        cur.execute('SELECT DISTINCT partition FROM country_name')
+        partitions = set([0])
+        for row in cur:
+            partitions.add(row[0])
+
+    return partitions
+
+
+def _get_tables(conn):
+    """ Return the set of tables currently in use.
+        Only includes non-partitioned
+    """
+    with conn.cursor() as cur:
+        cur.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
+
+        return set((row[0] for row in list(cur)))
+
+
+def _setup_tablespace_sql(config):
+    """ Returns a dict with tablespace expressions for the different tablespace
+        kinds depending on whether a tablespace is configured or not.
+    """
+    out = {}
+    for subset in ('ADDRESS', 'SEARCH', 'AUX'):
+        for kind in ('DATA', 'INDEX'):
+            tspace = getattr(config, 'TABLESPACE_{}_{}'.format(subset, kind))
+            if tspace:
+                tspace = 'TABLESPACE "{}"'.format(tspace)
+            out['{}_{}'.format(subset.lower, kind.lower())] = tspace
+
+    return out
+
+
+def _setup_postgres_sql(conn):
+    """ Set up a dictionary with various Postgresql/Postgis SQL terms which
+        are dependent on the database version in use.
+    """
+    out = {}
+    pg_version = conn.server_version_tuple()
+    # CREATE INDEX IF NOT EXISTS was introduced in PG9.5.
+    # Note that you need to ignore failures on older versions when
+    # unsing this construct.
+    out['if_index_not_exists'] = ' IF NOT EXISTS ' if pg_version >= (9, 5, 0) else ''
+
+    return out
+
+
+class SQLPreprocessor: # pylint: disable=too-few-public-methods
+    """ A environment for preprocessing SQL files from the
+        lib-sql directory.
+
+        The preprocessor provides a number of default filters and variables.
+        The variables may be overwritten when rendering an SQL file.
+
+        The preprocessing is currently based on the jinja2 templating library
+        and follows its syntax.
+    """
+
+    def __init__(self, conn, config, sqllib_dir):
+        self.env = jinja2.Environment(autoescape=False,
+                                      loader=jinja2.FileSystemLoader(str(sqllib_dir)))
+
+        db_info = {}
+        db_info['partitions'] = _get_partitions(conn)
+        db_info['tables'] = _get_tables(conn)
+        db_info['reverse_only'] = 'search_name' not in db_info['tables']
+        db_info['tablespace'] = _setup_tablespace_sql(config)
+
+        self.env.globals['config'] = config
+        self.env.globals['db'] = db_info
+        self.env.globals['sql'] = _setup_postgres_sql(conn)
+        self.env.globals['modulepath'] = config.DATABASE_MODULE_PATH or \
+                                         str((config.project_dir / 'module').resolve())
+
+
+    def run_sql_file(self, conn, name, **kwargs):
+        """ Execute the given SQL file on the connection. The keyword arguments
+            may supply additional parameters for preprocessing.
+        """
+        sql = self.env.get_template(name).render(**kwargs)
+
+        with conn.cursor() as cur:
+            cur.execute(sql)
+        conn.commit()

nominatim/tools/database_import.py

@@ -14,6 +14,7 @@ import psycopg2
 from ..db.connection import connect, get_pg_env
 from ..db import utils as db_utils
 from ..db.async_connection import DBConnection
+from ..db.sql_preprocessor import SQLPreprocessor
 from .exec_utils import run_osm2pgsql
 from ..errors import UsageError
 from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
@@ -178,6 +179,32 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False):
             Path(options['flatnode_file']).unlink()
 
 
+def create_tables(conn, config, sqllib_dir, reverse_only=False):
+    """ Create the set of basic tables.
+        When `reverse_only` is True, then the main table for searching will
+        be skipped and only reverse search is possible.
+    """
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+    sql.env.globals['db']['reverse_only'] = reverse_only
+
+    sql.run_sql_file(conn, 'tables.sql')
+
+
+def create_table_triggers(conn, config, sqllib_dir):
+    """ Create the triggers for the tables. The trigger functions must already
+        have been imported with refresh.create_functions().
+    """
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+    sql.run_sql_file(conn, 'table-triggers.sql')
+
+
+def create_partition_tables(conn, config, sqllib_dir):
+    """ Create tables that have explicit partitioning.
+    """
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+    sql.run_sql_file(conn, 'partition-tables.src.sql')
+
+
 def truncate_data_tables(conn, max_word_frequency=None):
     """ Truncate all data tables to prepare for a fresh load.
     """
@@ -258,3 +285,24 @@ def load_data(dsn, data_dir, threads):
     with connect(dsn) as conn:
         with conn.cursor() as cur:
             cur.execute('ANALYSE')
+
+
+def create_search_indices(conn, config, sqllib_dir, drop=False):
+    """ Create tables that have explicit partitioning.
+    """
+
+    # If index creation failed and left an index invalid, they need to be
+    # cleaned out first, so that the script recreates them.
+    with conn.cursor() as cur:
+        cur.execute("""SELECT relname FROM pg_class, pg_index
+                       WHERE pg_index.indisvalid = false
+                             AND pg_index.indexrelid = pg_class.oid""")
+        bad_indices = [row[0] for row in list(cur)]
+        for idx in bad_indices:
+            LOG.info("Drop invalid index %s.", idx)
+            cur.execute('DROP INDEX "{}"'.format(idx))
+    conn.commit()
+
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+
+    sql.run_sql_file(conn, 'indices.sql', drop=drop)

nominatim/tools/refresh.py

@@ -3,12 +3,12 @@ Functions for bringing auxiliary data in the database up-to-date.
 """
 import json
 import logging
-import re
 from textwrap import dedent
 
 from psycopg2.extras import execute_values
 
 from ..db.utils import execute_file
+from ..db.sql_preprocessor import SQLPreprocessor
 from ..version import NOMINATIM_VERSION
 
 LOG = logging.getLogger()
@@ -76,100 +76,17 @@ def load_address_levels_from_file(conn, config_file):
     with config_file.open('r') as fdesc:
         load_address_levels(conn, 'address_levels', json.load(fdesc))
 
-PLPGSQL_BASE_MODULES = (
-    'utils.sql',
-    'normalization.sql',
-    'ranking.sql',
-    'importance.sql',
-    'address_lookup.sql',
-    'interpolation.sql'
-)
 
-PLPGSQL_TABLE_MODULES = (
+def create_functions(conn, config, sqllib_dir,
-    ('place', 'place_triggers.sql'),
-    ('placex', 'placex_triggers.sql'),
-    ('location_postcode', 'postcode_triggers.sql')
-)
-
-def _get_standard_function_sql(conn, config, sql_dir, enable_diff_updates, enable_debug):
-    """ Read all applicable SQLs containing PL/pgSQL functions, replace
-        placefolders and execute them.
-    """
-    sql_func_dir = sql_dir / 'functions'
-    sql = ''
-
-    # Get the basic set of functions that is always imported.
-    for sql_file in PLPGSQL_BASE_MODULES:
-        with (sql_func_dir / sql_file).open('r') as fdesc:
-            sql += fdesc.read()
-
-    # Some files require the presence of a certain table
-    for table, fname in PLPGSQL_TABLE_MODULES:
-        if conn.table_exists(table):
-            with (sql_func_dir / fname).open('r') as fdesc:
-                sql += fdesc.read()
-
-    # Replace placeholders.
-    sql = sql.replace('{modulepath}',
-                      config.DATABASE_MODULE_PATH or str((config.project_dir / 'module').resolve()))
-
-    if enable_diff_updates:
-        sql = sql.replace('RETURN NEW; -- %DIFFUPDATES%', '--')
-
-    if enable_debug:
-        sql = sql.replace('--DEBUG:', '')
-
-    if config.get_bool('LIMIT_REINDEXING'):
-        sql = sql.replace('--LIMIT INDEXING:', '')
-
-    if not config.get_bool('USE_US_TIGER_DATA'):
-        sql = sql.replace('-- %NOTIGERDATA% ', '')
-
-    if not config.get_bool('USE_AUX_LOCATION_DATA'):
-        sql = sql.replace('-- %NOAUXDATA% ', '')
-
-    reverse_only = 'false' if conn.table_exists('search_name') else 'true'
-
-    return sql.replace('%REVERSE-ONLY%', reverse_only)
-
-
-def replace_partition_string(sql, partitions):
-    """ Replace a partition template with the actual partition code.
-    """
-    for match in re.findall('^-- start(.*?)^-- end', sql, re.M | re.S):
-        repl = ''
-        for part in partitions:
-            repl += match.replace('-partition-', str(part))
-        sql = sql.replace(match, repl)
-
-    return sql
-
-def _get_partition_function_sql(conn, sql_dir):
-    """ Create functions that work on partition tables.
-    """
-    with conn.cursor() as cur:
-        cur.execute('SELECT distinct partition FROM country_name')
-        partitions = set([0])
-        for row in cur:
-            partitions.add(row[0])
-
-    with (sql_dir / 'partition-functions.src.sql').open('r') as fdesc:
-        sql = fdesc.read()
-
-    return replace_partition_string(sql, sorted(partitions))
-
-def create_functions(conn, config, sql_dir,
                      enable_diff_updates=True, enable_debug=False):
     """ (Re)create the PL/pgSQL functions.
     """
-    sql = _get_standard_function_sql(conn, config, sql_dir,
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
-                                     enable_diff_updates, enable_debug)
-    sql += _get_partition_function_sql(conn, sql_dir)
 
-    with conn.cursor() as cur:
+    sql.run_sql_file(conn, 'functions.sql',
-        cur.execute(sql)
+                     disable_diff_update=not enable_diff_updates,
+                     debug=enable_debug)
 
-    conn.commit()
 
 
 WEBSITE_SCRIPTS = (

test/python/test_cli.py

@@ -92,6 +92,11 @@ def test_import_full(temp_db, mock_func_factory):
         mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
         mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
         mock_func_factory(nominatim.tools.database_import, 'load_data'),
+        mock_func_factory(nominatim.tools.database_import, 'create_tables'),
+        mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
+        mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
+        mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
+        mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'),
         mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
         mock_func_factory(nominatim.tools.refresh, 'setup_website'),
         mock_func_factory(nominatim.db.properties, 'set_property')

test/python/test_db_sql_preprocessor.py

@@ -0,0 +1,51 @@
+"""
+Tests for SQL preprocessing.
+"""
+from pathlib import Path
+
+import pytest
+
+from nominatim.db.sql_preprocessor import SQLPreprocessor
+
+@pytest.fixture
+def sql_factory(tmp_path):
+    def _mk_sql(sql_body):
+        (tmp_path / 'test.sql').write_text("""
+          CREATE OR REPLACE FUNCTION test() RETURNS TEXT
+          AS $$
+          BEGIN
+            {}
+          END;
+          $$ LANGUAGE plpgsql IMMUTABLE;""".format(sql_body))
+        return 'test.sql'
+
+    return _mk_sql
+
+
+@pytest.fixture
+def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
+    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
+    table_factory('country_name', 'partition INT', (0, 1, 2))
+    return SQLPreprocessor(temp_db_conn, def_config, tmp_path)
+
+@pytest.mark.parametrize("expr,ret", [
+    ("'a'", 'a'),
+    ("'{{db.partitions|join}}'", '012'),
+    ("{% if 'country_name' in db.tables %}'yes'{% else %}'no'{% endif %}", "yes"),
+    ("{% if 'xxx' in db.tables %}'yes'{% else %}'no'{% endif %}", "no"),
+    ("'{{config.DATABASE_MODULE_PATH}}'", '.')
+    ])
+def test_load_file_simple(sql_preprocessor, sql_factory, temp_db_conn, temp_db_cursor, expr, ret):
+    sqlfile = sql_factory("RETURN {};".format(expr))
+
+    sql_preprocessor.run_sql_file(temp_db_conn, sqlfile)
+
+    assert temp_db_cursor.scalar('SELECT test()') == ret
+
+
+def test_load_file_with_params(sql_preprocessor, sql_factory, temp_db_conn, temp_db_cursor):
+    sqlfile = sql_factory("RETURN '{{ foo }} {{ bar }}';")
+
+    sql_preprocessor.run_sql_file(temp_db_conn, sqlfile, bar='XX', foo='ZZ')
+
+    assert temp_db_cursor.scalar('SELECT test()') == 'ZZ XX'

test/python/test_tools_refresh_create_functions.py

@@ -1,98 +1,47 @@
 """
 Tests for creating PL/pgSQL functions for Nominatim.
 """
-from pathlib import Path
 import pytest
 
-from nominatim.db.connection import connect
+from nominatim.tools.refresh import create_functions
-from nominatim.tools.refresh import _get_standard_function_sql, _get_partition_function_sql
-
-SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve()
 
 @pytest.fixture
-def db(temp_db):
+def conn(temp_db_conn, table_factory, monkeypatch):
-    with connect('dbname=' + temp_db) as conn:
+    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
-        yield conn
+    table_factory('country_name', 'partition INT', (0, 1, 2))
-
+    return temp_db_conn
-@pytest.fixture
-def db_with_tables(db):
-    with db.cursor() as cur:
-        for table in ('place', 'placex', 'location_postcode'):
-            cur.execute('CREATE TABLE {} (place_id BIGINT)'.format(table))
-
-    return db
 
 
-def test_standard_functions_replace_module_default(db, def_config):
+def test_create_functions(temp_db_cursor, conn, def_config, tmp_path):
-    def_config.project_dir = Path('.')
+    sqlfile = tmp_path / 'functions.sql'
-    sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False)
+    sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER
+                          AS $$
+                          BEGIN
+                            RETURN 43;
+                          END;
+                          $$ LANGUAGE plpgsql IMMUTABLE;
+                       """)
 
-    assert sql
+    create_functions(conn, def_config, tmp_path)
-    assert sql.find('{modulepath}') < 0
+
-    assert sql.find("'{}'".format(Path('module/nominatim.so').resolve())) >= 0
+    assert temp_db_cursor.scalar('SELECT test()') == 43
 
 
-def test_standard_functions_replace_module_custom(monkeypatch, db, def_config):
+@pytest.mark.parametrize("dbg,ret", ((True, 43), (False, 22)))
-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', 'custom')
+def test_create_functions_with_template(temp_db_cursor, conn, def_config, tmp_path, dbg, ret):
-    sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False)
+    sqlfile = tmp_path / 'functions.sql'
+    sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER
+                          AS $$
+                          BEGIN
+                            {% if debug %}
+                            RETURN 43;
+                            {% else %}
+                            RETURN 22;
+                            {% endif %}
+                          END;
+                          $$ LANGUAGE plpgsql IMMUTABLE;
+                       """)
 
-    assert sql
+    create_functions(conn, def_config, tmp_path, enable_debug=dbg)
-    assert sql.find('{modulepath}') < 0
-    assert sql.find("'custom/nominatim.so'") >= 0
 
-
+    assert temp_db_cursor.scalar('SELECT test()') == ret
-@pytest.mark.parametrize("enabled", (True, False))
-def test_standard_functions_enable_diff(db_with_tables, def_config, enabled):
-    def_config.project_dir = Path('.')
-    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, enabled, False)
-
-    assert sql
-    assert (sql.find('%DIFFUPDATES%') < 0) == enabled
-
-
-@pytest.mark.parametrize("enabled", (True, False))
-def test_standard_functions_enable_debug(db_with_tables, def_config, enabled):
-    def_config.project_dir = Path('.')
-    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, enabled)
-
-    assert sql
-    assert (sql.find('--DEBUG') < 0) == enabled
-
-
-@pytest.mark.parametrize("enabled", (True, False))
-def test_standard_functions_enable_limit_reindexing(monkeypatch, db_with_tables, def_config, enabled):
-    def_config.project_dir = Path('.')
-    monkeypatch.setenv('NOMINATIM_LIMIT_REINDEXING', 'yes' if enabled else 'no')
-    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
-
-    assert sql
-    assert (sql.find('--LIMIT INDEXING') < 0) == enabled
-
-
-@pytest.mark.parametrize("enabled", (True, False))
-def test_standard_functions_enable_tiger(monkeypatch, db_with_tables, def_config, enabled):
-    def_config.project_dir = Path('.')
-    monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA', 'yes' if enabled else 'no')
-    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
-
-    assert sql
-    assert (sql.find('%NOTIGERDATA%') >= 0) == enabled
-
-
-@pytest.mark.parametrize("enabled", (True, False))
-def test_standard_functions_enable_aux(monkeypatch, db_with_tables, def_config, enabled):
-    def_config.project_dir = Path('.')
-    monkeypatch.setenv('NOMINATIM_USE_AUX_LOCATION_DATA', 'yes' if enabled else 'no')
-    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
-
-    assert sql
-    assert (sql.find('%NOAUXDATA%') >= 0) == enabled
-
-
-def test_partition_function(temp_db_cursor, db, def_config):
-    temp_db_cursor.execute("CREATE TABLE country_name (partition SMALLINT)")
-
-    sql = _get_partition_function_sql(db, SQL_DIR)
-
-    assert sql
-    assert sql.find('-partition-') < 0

vagrant/Install-on-Centos-7.sh

@@ -42,7 +42,7 @@
                         python3-pip python3-setuptools python3-devel \
                         expat-devel zlib-devel
 
-    pip3 install --user psycopg2 python-dotenv psutil
+    pip3 install --user psycopg2 python-dotenv psutil Jinja2
 
 
 #

vagrant/Install-on-Centos-8.sh

@@ -35,7 +35,7 @@
                         python3-pip python3-setuptools python3-devel \
                         expat-devel zlib-devel
 
-    pip3 install --user psycopg2 python-dotenv psutil
+    pip3 install --user psycopg2 python-dotenv psutil Jinja2
 
 
 #

vagrant/Install-on-Ubuntu-18.sh

@@ -30,7 +30,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-server-dev-10 postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php php-pgsql php-intl python3-pip \
-                        python3-psycopg2 python3-psutil git
+                        python3-psycopg2 python3-psutil python3-jinja2 git
 
 # The python-dotenv package that comes with Ubuntu 18.04 is too old, so
 # install the latest version from pip:

vagrant/Install-on-Ubuntu-20.sh

@@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-server-dev-12 postgresql-12-postgis-3 \
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php php-pgsql php-intl python3-dotenv \
-                        python3-psycopg2 python3-psutil git
+                        python3-psycopg2 python3-psutil python3-jinja2 git
 
 #
 # System Configuration