safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

remove website setup

Browse Source 
The website directory was for PHP scripts only and is no longer
needed.
This commit is contained in:
Sarah Hoffmann
2024-09-15 11:58:55 +02:00
parent 06683edaae
commit 6bc044d9c7
7 changed files with 6 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/nominatim_db/clicmd/refresh.py
View File

@@ -69,7 +69,8 @@ class UpdateRefresh:
group.add_argument('--importance', action='store_true',
help='Recompute place importances (expensive!)')
group.add_argument('--website', action='store_true',
help='Refresh the directory that serves the scripts for the web API')
help='DEPRECATED. This function has no function anymore'
' and will be removed in a future version.')
group.add_argument('--data-object', action='append',
type=_parse_osm_object, metavar='OBJECT',
help='Mark the given OSM object as requiring an update'
@@ -159,14 +160,8 @@ class UpdateRefresh:
refresh.recompute_importance(conn)
if args.website:
webdir = args.project_dir / 'website'
LOG.warning('Setting up website directory at %s', webdir)
# This is a little bit hacky: call the tokenizer setup, so that
# the tokenizer directory gets repopulated as well, in case it
# wasn't there yet.
self._get_tokenizer(args.config)
with connect(args.config.get_libpq_dsn()) as conn:
refresh.setup_website(webdir, args.config, conn)
LOG.error('WARNING: Website setup is no longer required. '
'This function will be removed in future version of Nominatim.')
if args.data_object or args.data_area:
with connect(args.config.get_libpq_dsn()) as conn:

7
src/nominatim_db/clicmd/setup.py
View File

@@ -88,7 +88,7 @@ class SetupAll:
async def async_run(self, args: NominatimArgs) -> int:
from ..data import country_info
from ..tools import database_import, refresh, postcodes, freeze
from ..tools import database_import, postcodes, freeze
from ..indexer.indexer import Indexer
num_threads = args.threads or psutil.cpu_count() or 1
@@ -141,11 +141,6 @@ class SetupAll:
LOG.warning('Recompute word counts')
tokenizer.update_statistics(args.config, threads=num_threads)
webdir = args.project_dir / 'website'
LOG.warning('Setup website at %s', webdir)
with connect(args.config.get_libpq_dsn()) as conn:
refresh.setup_website(webdir, args.config, conn)
self._finalize_database(args.config.get_libpq_dsn(), args.offline)
return 0

72
src/nominatim_db/tools/refresh.py
View File

@@ -11,17 +11,15 @@ from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, c
import csv
import gzip
import logging
from textwrap import dedent
from pathlib import Path
from psycopg import sql as pysql
from ..config import Configuration
from ..db.connection import Connection, connect, postgis_version_tuple,\
drop_tables, table_exists
drop_tables
from ..db.utils import execute_file
from ..db.sql_preprocessor import SQLPreprocessor
from ..version import NOMINATIM_VERSION
LOG = logging.getLogger()
@@ -99,34 +97,6 @@ def create_functions(conn: Connection, config: Configuration,
debug=enable_debug)
WEBSITE_SCRIPTS = (
'deletable.php',
'details.php',
'lookup.php',
'polygons.php',
'reverse.php',
'search.php',
'status.php'
)
# constants needed by PHP scripts: PHP name, config name, type
PHP_CONST_DEFS = (
('Database_DSN', 'DATABASE_DSN', str),
('Default_Language', 'DEFAULT_LANGUAGE', str),
('Log_DB', 'LOG_DB', bool),
('Log_File', 'LOG_FILE', Path),
('NoAccessControl', 'CORS_NOACCESSCONTROL', bool),
('Places_Max_ID_count', 'LOOKUP_MAX_COUNT', int),
('PolygonOutput_MaximumTypes', 'POLYGON_OUTPUT_MAX_TYPES', int),
('Search_BatchMode', 'SEARCH_BATCH_MODE', bool),
('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
('MapIcon_URL', 'MAPICON_URL', str),
('Search_WithinCountries', 'SEARCH_WITHIN_COUNTRIES', bool),
)
def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
""" Replaces the wikipedia importance tables with new data.
The import is run in a single transaction so that the new data
@@ -272,46 +242,6 @@ def _quote_php_variable(var_type: Type[Any], config: Configuration,
return f"'{quoted}'"
def setup_website(basedir: Path, config: Configuration, conn: Connection) -> None:
""" Create the website script stubs.
"""
if config.lib_dir.php is None:
LOG.info("Python frontend does not require website setup. Skipping.")
return
if not basedir.exists():
LOG.info('Creating website directory.')
basedir.mkdir()
assert config.project_dir is not None
basedata = dedent(f"""\
<?php
@define('CONST_Debug', $_GET['debug'] ?? false);
@define('CONST_LibDir', '{config.lib_dir.php}');
@define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
@define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');
""")
for php_name, conf_name, var_type in PHP_CONST_DEFS:
varout = _quote_php_variable(var_type, config, conf_name)
basedata += f"@define('CONST_{php_name}', {varout});\n"
template = "\nrequire_once(CONST_LibDir.'/website/{}');\n"
search_name_table_exists = bool(conn and table_exists(conn, 'search_name'))
for script in WEBSITE_SCRIPTS:
if not search_name_table_exists and script == 'search.php':
out = template.format('reverse-only-search.php')
else:
out = template.format(script)
(basedir / script).write_text(basedata + out, 'utf-8')
def invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,
recursive: bool = True) -> None:
""" Mark the given OSM object for reindexing. When 'recursive' is set

9
test/bdd/steps/nominatim_environment.py
View File

@@ -110,15 +110,6 @@ class NominatimEnvironment:
self.website_dir = tempfile.TemporaryDirectory()
try:
conn = self.connect_database(dbname)
except:
conn = False
refresh.setup_website(Path(self.website_dir.name) / 'website',
self.get_test_config(), conn)
if conn:
conn.close()
def get_test_config(self):
cfg = Configuration(Path(self.website_dir.name), environ=self.test_env)

4
test/python/cli/test_cmd_import.py
View File

@@ -52,7 +52,6 @@ class TestCliImportWithDb:
mock_func_factory(nominatim_db.tools.refresh, 'load_address_levels_from_config'),
mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes'),
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
]
params = ['import', '--osm-file', __file__]
@@ -81,7 +80,6 @@ class TestCliImportWithDb:
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes'),
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
mock_func_factory(nominatim_db.db.properties, 'set_property')
]
@@ -98,7 +96,6 @@ class TestCliImportWithDb:
async_mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
mock_func_factory(nominatim_db.db.properties, 'set_property')
]
@@ -115,7 +112,6 @@ class TestCliImportWithDb:
mocks = [
async_mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
mock_func_factory(nominatim_db.db.properties, 'set_property')
]

1
test/python/cli/test_cmd_refresh.py
View File

@@ -25,7 +25,6 @@ class TestRefresh:
('address-levels', 'load_address_levels_from_config'),
('wiki-data', 'import_wikipedia_articles'),
('importance', 'recompute_importance'),
('website', 'setup_website'),
])
def test_refresh_command(self, mock_func_factory, command, func):
mock_func_factory(nominatim_db.tools.refresh, 'create_functions')

104
test/python/tools/test_refresh_setup_website.py
View File

@@ -1,104 +0,0 @@
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Tests for setting up the website scripts.
"""
import subprocess
import pytest
from nominatim_db.tools import refresh
@pytest.fixture
def test_script(tmp_path):
(tmp_path / 'php').mkdir()
website_dir = (tmp_path / 'php' / 'website')
website_dir.mkdir()
def _create_file(code):
outfile = website_dir / 'reverse-only-search.php'
outfile.write_text('<?php\n{}\n'.format(code), 'utf-8')
return _create_file
@pytest.fixture
def run_website_script(tmp_path, project_env, temp_db_conn):
project_env.lib_dir.php = tmp_path / 'php'
def _runner():
refresh.setup_website(tmp_path, project_env, temp_db_conn)
proc = subprocess.run(['/usr/bin/env', 'php', '-Cq',
tmp_path / 'search.php'], check=False)
return proc.returncode
return _runner
def test_basedir_created(tmp_path, project_env, temp_db_conn):
webdir = tmp_path / 'website'
assert not webdir.exists()
refresh.setup_website(webdir, project_env, temp_db_conn)
assert webdir.exists()
@pytest.mark.parametrize("setting,retval", (('yes', 10), ('no', 20)))
def test_setup_website_check_bool(monkeypatch, test_script, run_website_script,
setting, retval):
monkeypatch.setenv('NOMINATIM_CORS_NOACCESSCONTROL', setting)
test_script('exit(CONST_NoAccessControl ? 10 : 20);')
assert run_website_script() == retval
@pytest.mark.parametrize("setting", (0, 10, 99067))
def test_setup_website_check_int(monkeypatch, test_script, run_website_script, setting):
monkeypatch.setenv('NOMINATIM_LOOKUP_MAX_COUNT', str(setting))
test_script('exit(CONST_Places_Max_ID_count == {} ? 10 : 20);'.format(setting))
assert run_website_script() == 10
def test_setup_website_check_empty_str(monkeypatch, test_script, run_website_script):
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', '')
test_script('exit(CONST_Default_Language === false ? 10 : 20);')
assert run_website_script() == 10
def test_setup_website_check_str(monkeypatch, test_script, run_website_script):
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'ffde 2')
test_script('exit(CONST_Default_Language === "ffde 2" ? 10 : 20);')
assert run_website_script() == 10
def test_relative_log_file(project_env, monkeypatch, test_script, run_website_script):
monkeypatch.setenv('NOMINATIM_LOG_FILE', 'access.log')
expected_file = str(project_env.project_dir / 'access.log')
test_script(f'exit(CONST_Log_File === "{expected_file}" ? 10 : 20);')
assert run_website_script() == 10
def test_variable_with_bracket(project_env, monkeypatch, test_script, run_website_script):
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=nominatim;user=foo;password=4{5')
test_script('exit(CONST_Database_DSN === "pgsql:dbname=nominatim;user=foo;password=4{5" ? 10 : 20);')
assert run_website_script() == 10