From 2197236872dfc8a0739d188251a01f3b5811550a Mon Sep 17 00:00:00 2001 From: Sri CHaRan Date: Tue, 30 Dec 2025 00:07:49 +0530 Subject: [PATCH 1/5] Add experimental Windows CI workflow --- .github/workflows/ci-tests-windows.yml | 44 ++++++++++++++++++++++++++ .github/workflows/windows-ci.yml | 34 ++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 .github/workflows/ci-tests-windows.yml create mode 100644 .github/workflows/windows-ci.yml diff --git a/.github/workflows/ci-tests-windows.yml b/.github/workflows/ci-tests-windows.yml new file mode 100644 index 00000000..6ca82a1f --- /dev/null +++ b/.github/workflows/ci-tests-windows.yml @@ -0,0 +1,44 @@ +name: CI Tests (Windows) + +on: [ push, pull_request ] + +jobs: + tests: + strategy: + matrix: + flavour: ["windows-latest"] + include: + - flavour: windows-latest + runner: windows-latest + python: '3.11' + + runs-on: ${{ matrix.runner }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + + - name: Install test prerequisites + run: | + python -m pip install -U pip + python -m pip install \ + pytest pytest-asyncio \ + "psycopg[binary]!=3.3.0" \ + python-dotenv pyyaml jinja2 psutil sqlalchemy + + - name: Python unit tests (Windows subset) + run: | + python -m pytest \ + test/python/utils \ + test/python/config \ + test/python/api/test_api_types.py \ + test/python/api/search/test_query.py \ + test/python/api/query_processing/test_regex_replace.py \ + test/python/api/query_processing/test_split_japanese_phrases.py \ No newline at end of file diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml new file mode 100644 index 00000000..be44f5a9 --- /dev/null +++ b/.github/workflows/windows-ci.yml @@ -0,0 +1,34 @@ +name: Windows CI (experimental) + +on: + pull_request: + push: + +jobs: + windows-smoke: + runs-on: windows-latest + continue-on-error: true #this is a smoke test (experimental); it won't block merges + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + packaging/nominatim-db/pyproject.toml + packaging/nominatim-api/pyproject.toml + + - name: Install latest flake8 + run: python -m pip install -U pip flake8 + + - name: Python linting + run: python -m flake8 src test/python test/bdd + + - name: Compile Python sources (smoke) + run: python -m compileall src/nominatim_db src/nominatim_api nominatim-cli.py From abd7c302f8bfe0c6b2bf0a4884ae9ebe29c03bb5 Mon Sep 17 00:00:00 2001 From: Sri Charan Chittineni Date: Sat, 17 Jan 2026 10:25:44 +0530 Subject: [PATCH 2/5] implement stage 1 : python unit tests --- .github/workflows/ci-tests.yml | 27 +++++++++++++++++++++++++++ test/python/config/test_config.py | 7 ++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index ff89bfbf..9b6c6b99 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -140,6 +140,33 @@ jobs: ../venv/bin/python -m pytest test/bdd --nominatim-purge working-directory: Nominatim + tests-windows: + needs: create-archive + runs-on: windows-latest + + steps: + - uses: actions/download-artifact@v4 + with: + name: full-source + + - name: Unpack Nominatim + run: tar xf nominatim-src.tar.bz2 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install test prerequisites + run: | + python -m pip install -U pip + python -m pip install pytest pytest-asyncio "psycopg[binary]!=3.3.0" python-dotenv pyyaml jinja2 psutil sqlalchemy + + - name: Python unit tests (Windows subset) + run: | + python -m pytest test/python/utils test/python/config test/python/api/test_api_types.py test/python/api/search/test_query.py test/python/api/query_processing/test_regex_replace.py test/python/api/query_processing/test_split_japanese_phrases.py + working-directory: Nominatim + install: runs-on: ubuntu-latest needs: create-archive diff --git a/test/python/config/test_config.py b/test/python/config/test_config.py index 555bc4e7..4908fd9a 100644 --- a/test/python/config/test_config.py +++ b/test/python/config/test_config.py @@ -200,14 +200,15 @@ def test_get_path_empty(make_config): assert not config.get_path('TOKENIZER_CONFIG') -def test_get_path_absolute(make_config, monkeypatch): +def test_get_path_absolute(make_config, monkeypatch, tmp_path): config = make_config() - monkeypatch.setenv('NOMINATIM_FOOBAR', '/dont/care') + p = (tmp_path / "does_not_exist").resolve() + monkeypatch.setenv('NOMINATIM_FOOBAR', str(p)) result = config.get_path('FOOBAR') assert isinstance(result, Path) - assert str(result) == '/dont/care' + assert str(result) == str(p) def test_get_path_relative(make_config, monkeypatch, tmp_path): From 238f3dd1d940f43ede8d07d43c7d4bec6990952d Mon Sep 17 00:00:00 2001 From: Sri CHaRan Date: Thu, 29 Jan 2026 21:49:59 +0530 Subject: [PATCH 3/5] ci/windows: add Postgresql setup action to tests --- .../setup-postgresql-windows/action.yml | 83 +++++++++++++++++++ .github/actions/setup-postgresql/action.yml | 2 + .github/workflows/ci-tests.yml | 4 + 3 files changed, 89 insertions(+) create mode 100644 .github/actions/setup-postgresql-windows/action.yml diff --git a/.github/actions/setup-postgresql-windows/action.yml b/.github/actions/setup-postgresql-windows/action.yml new file mode 100644 index 00000000..41256435 --- /dev/null +++ b/.github/actions/setup-postgresql-windows/action.yml @@ -0,0 +1,83 @@ +name: 'Setup Postgresql and Postgis on Windows' + +description: 'Installs PostgreSQL and PostGIS for Windows and configures it for CI tests' + +inputs: + postgresql-version: + description: 'Version of PostgreSQL to install' + required: true + +runs: + using: "composite" + + steps: + - name: Decide Postgis version (Windows) + id: postgis-ver + shell: pwsh + run: | + echo "PowerShell version: ${PSVersionTable.PSVersion}" + $PG_VERSION = Split-Path $env:PGROOT -Leaf + $postgis_page = "https://download.osgeo.org/postgis/windows/pg$PG_VERSION" + echo "Detecting PostGIS version from $postgis_page for PostgreSQL $PG_VERSION" + $pgis_bundle = (Invoke-WebRequest -Uri $postgis_page -ErrorAction Stop).Links.Where({$_.href -match "^postgis.*zip$"}).href + if (!$pgis_bundle) { + Write-Error "Could not find latest PostGIS version in $postgis_page that would match ^postgis.*zip$ pattern" + exit 1 + } + $pgis_bundle = [IO.Path]::ChangeExtension($pgis_bundle, [NullString]::Value) + $pgis_bundle_url = "$postgis_page/$pgis_bundle.zip" + Add-Content $env:GITHUB_OUTPUT "postgis_file=$pgis_bundle" + Add-Content $env:GITHUB_OUTPUT "postgis_bundle_url=$pgis_bundle_url" + + - uses: actions/cache@v4 + with: + path: | + C:/postgis.zip + key: postgis-cache-${{ steps.postgis-ver.outputs.postgis_file }} + + - name: Download postgis + run: | + if (!(Test-Path "C:\postgis.zip")){(new-object net.webclient).DownloadFile($env:PGIS_BUNDLE_URL, "c:\postgis.zip")} + if (Test-path "c:\postgis_archive"){Remove-Item "c:\postgis_archive" -Recurse -Force} + 7z x c:\postgis.zip -oc:\postgis_archive + shell: pwsh + env: + PGIS_BUNDLE_URL: ${{ steps.postgis-ver.outputs.postgis_bundle_url }} + + - name: Install postgis + run: | + echo "Root: $PGROOT, Bin: $PGBIN" + cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT" + shell: bash + + - name: Start PostgreSQL on Windows + run: | + $pgService = Get-Service -Name postgresql* + Set-Service -InputObject $pgService -Status running -StartupType automatic + Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru + shell: pwsh + + - name: Adapt postgresql configuration + shell: pwsh + env: + PGPASSWORD: root + run: | + & "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET fsync = 'off';" + & "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET synchronous_commit = 'off';" + & "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET full_page_writes = 'off';" + & "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET shared_buffers = '1GB';" + & "$env:PGBIN\psql" -U postgres -d postgres -c "ALTER SYSTEM SET port = 5432;" + + Restart-Service -Name postgresql* + Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru + + - name: Setup database users + shell: pwsh + env: + PGPASSWORD: root + run: | + & "$env:PGBIN\createuser" -U postgres -S www-data + & "$env:PGBIN\createuser" -U postgres -s runner + + + diff --git a/.github/actions/setup-postgresql/action.yml b/.github/actions/setup-postgresql/action.yml index 7a9590c1..b868546b 100644 --- a/.github/actions/setup-postgresql/action.yml +++ b/.github/actions/setup-postgresql/action.yml @@ -1,5 +1,7 @@ name: 'Setup Postgresql and Postgis' +description: 'Installs PostgreSQL and PostGIS and configures it for CI tests' + inputs: postgresql-version: description: 'Version of PostgreSQL to install' diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9b6c6b99..3d2d1090 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -152,6 +152,10 @@ jobs: - name: Unpack Nominatim run: tar xf nominatim-src.tar.bz2 + - uses: ./Nominatim/.github/actions/setup-postgresql-windows + with: + postgresql-version: 16 + - name: Set up Python uses: actions/setup-python@v5 with: From bf5ef0140a927f7e35043f7ef08259be1747ef02 Mon Sep 17 00:00:00 2001 From: Sri CHaRan Date: Mon, 2 Feb 2026 08:23:48 +0530 Subject: [PATCH 4/5] ci/windows: enable full python unit test setup for windows --- .github/workflows/ci-tests.yml | 32 +++++++++++++++++++++++++++----- test/python/conftest.py | 4 ++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 3d2d1090..4d6608c5 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -154,21 +154,43 @@ jobs: - uses: ./Nominatim/.github/actions/setup-postgresql-windows with: - postgresql-version: 16 + postgresql-version: 18 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.14' + + - name: Install Spatialite + run: | + Invoke-WebRequest -Uri "https://www.gaia-gis.it/gaia-sins/windows-bin-amd64/mod_spatialite-5.1.0-win-amd64.7z" -OutFile "spatialite.7z" + 7z x spatialite.7z -o"C:\spatialite" + echo "C:\spatialite\mod_spatialite-5.1.0-win-amd64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Set UTF-8 encoding + run: | + echo "PYTHONUTF8=1" >> $env:GITHUB_ENV + [System.Console]::OutputEncoding = [System.Text.Encoding]::UTF8 + + - name: Install PyICU from wheel + run: | + python -m pip install https://github.com/cgohlke/pyicu-build/releases/download/v2.16.0/pyicu-2.16-cp314-cp314-win_amd64.whl - name: Install test prerequisites run: | python -m pip install -U pip - python -m pip install pytest pytest-asyncio "psycopg[binary]!=3.3.0" python-dotenv pyyaml jinja2 psutil sqlalchemy + python -m pip install pytest pytest-asyncio "psycopg[binary]!=3.3.0" python-dotenv pyyaml jinja2 psutil sqlalchemy pytest-bdd falcon starlette uvicorn asgi_lifespan aiosqlite osmium mwparserfromhell - - name: Python unit tests (Windows subset) + - name: Python unit tests (subset) run: | - python -m pytest test/python/utils test/python/config test/python/api/test_api_types.py test/python/api/search/test_query.py test/python/api/query_processing/test_regex_replace.py test/python/api/query_processing/test_split_japanese_phrases.py + python -m pytest test/python ` + --ignore=test/python/tools/test_add_osm_data.py ` + --ignore=test/python/tools/test_database_import.py ` + --ignore=test/python/tools/test_exec_utils.py ` + --ignore=test/python/cli/test_cmd_import.py ` + --ignore=test/python/data/test_country_info.py ` + --ignore=test/python/db/test_utils.py ` + --ignore=test/python/tools/test_refresh.py working-directory: Nominatim install: diff --git a/test/python/conftest.py b/test/python/conftest.py index 891716d3..f0aafce1 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -6,8 +6,12 @@ # For a full list of authors see the git log. import itertools import sys +import asyncio from pathlib import Path +if sys.platform == 'win32': + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + import psycopg from psycopg import sql as pysql import pytest From 24ba9651ba945be049c5d7dbcc7fdb1dd2b5bdcd Mon Sep 17 00:00:00 2001 From: Sri CHaRan Date: Tue, 3 Feb 2026 23:36:37 +0530 Subject: [PATCH 5/5] ci/windows: install osm2pgsql binary and enable full unit tests suite --- .../setup-postgresql-windows/action.yml | 18 ++++++-- .github/workflows/ci-tests-windows.yml | 44 ------------------- .github/workflows/ci-tests.yml | 26 ++++++----- .github/workflows/windows-ci.yml | 34 -------------- 4 files changed, 31 insertions(+), 91 deletions(-) delete mode 100644 .github/workflows/ci-tests-windows.yml delete mode 100644 .github/workflows/windows-ci.yml diff --git a/.github/actions/setup-postgresql-windows/action.yml b/.github/actions/setup-postgresql-windows/action.yml index 41256435..fab18778 100644 --- a/.github/actions/setup-postgresql-windows/action.yml +++ b/.github/actions/setup-postgresql-windows/action.yml @@ -11,6 +11,18 @@ runs: using: "composite" steps: + - name: Set up PostgreSQL variables + shell: pwsh + run: | + $version = "${{ inputs.postgresql-version }}" + $root = "C:\Program Files\PostgreSQL\$version" + $bin = "$root\bin" + + echo "PGROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "PGBIN=$bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + echo "$bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Decide Postgis version (Windows) id: postgis-ver shell: pwsh @@ -36,19 +48,19 @@ runs: key: postgis-cache-${{ steps.postgis-ver.outputs.postgis_file }} - name: Download postgis + shell: pwsh run: | if (!(Test-Path "C:\postgis.zip")){(new-object net.webclient).DownloadFile($env:PGIS_BUNDLE_URL, "c:\postgis.zip")} if (Test-path "c:\postgis_archive"){Remove-Item "c:\postgis_archive" -Recurse -Force} 7z x c:\postgis.zip -oc:\postgis_archive - shell: pwsh env: PGIS_BUNDLE_URL: ${{ steps.postgis-ver.outputs.postgis_bundle_url }} - name: Install postgis + shell: bash run: | echo "Root: $PGROOT, Bin: $PGBIN" - cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT" - shell: bash + cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT" - name: Start PostgreSQL on Windows run: | diff --git a/.github/workflows/ci-tests-windows.yml b/.github/workflows/ci-tests-windows.yml deleted file mode 100644 index 6ca82a1f..00000000 --- a/.github/workflows/ci-tests-windows.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: CI Tests (Windows) - -on: [ push, pull_request ] - -jobs: - tests: - strategy: - matrix: - flavour: ["windows-latest"] - include: - - flavour: windows-latest - runner: windows-latest - python: '3.11' - - runs-on: ${{ matrix.runner }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - cache: pip - - - name: Install test prerequisites - run: | - python -m pip install -U pip - python -m pip install \ - pytest pytest-asyncio \ - "psycopg[binary]!=3.3.0" \ - python-dotenv pyyaml jinja2 psutil sqlalchemy - - - name: Python unit tests (Windows subset) - run: | - python -m pytest \ - test/python/utils \ - test/python/config \ - test/python/api/test_api_types.py \ - test/python/api/search/test_query.py \ - test/python/api/query_processing/test_regex_replace.py \ - test/python/api/query_processing/test_split_japanese_phrases.py \ No newline at end of file diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4d6608c5..487753a7 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -154,7 +154,7 @@ jobs: - uses: ./Nominatim/.github/actions/setup-postgresql-windows with: - postgresql-version: 18 + postgresql-version: 17 - name: Set up Python uses: actions/setup-python@v5 @@ -167,6 +167,19 @@ jobs: 7z x spatialite.7z -o"C:\spatialite" echo "C:\spatialite\mod_spatialite-5.1.0-win-amd64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Install osm2pgsql + run: | + Invoke-WebRequest -Uri "https://osm2pgsql.org/download/windows/osm2pgsql-latest-x64.zip" -OutFile "osm2pgsql.zip" + Expand-Archive -Path "osm2pgsql.zip" -DestinationPath "C:\osm2pgsql" + $BinDir = Get-ChildItem -Path "C:\osm2pgsql" -Recurse -Filter "osm2pgsql.exe" | Select-Object -ExpandProperty DirectoryName | Select-Object -First 1 + if (-not $BinDir) { + Write-Error "Could not find osm2pgsql.exe" + exit 1 + } + echo "$BinDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + $FullExePath = Join-Path $BinDir "osm2pgsql.exe" + echo "NOMINATIM_OSM2PGSQL_BINARY=$FullExePath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Set UTF-8 encoding run: | echo "PYTHONUTF8=1" >> $env:GITHUB_ENV @@ -181,16 +194,9 @@ jobs: python -m pip install -U pip python -m pip install pytest pytest-asyncio "psycopg[binary]!=3.3.0" python-dotenv pyyaml jinja2 psutil sqlalchemy pytest-bdd falcon starlette uvicorn asgi_lifespan aiosqlite osmium mwparserfromhell - - name: Python unit tests (subset) + - name: Python unit tests run: | - python -m pytest test/python ` - --ignore=test/python/tools/test_add_osm_data.py ` - --ignore=test/python/tools/test_database_import.py ` - --ignore=test/python/tools/test_exec_utils.py ` - --ignore=test/python/cli/test_cmd_import.py ` - --ignore=test/python/data/test_country_info.py ` - --ignore=test/python/db/test_utils.py ` - --ignore=test/python/tools/test_refresh.py + python -m pytest test/python -k "not (import_osm or run_osm2pgsql)" working-directory: Nominatim install: diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml deleted file mode 100644 index be44f5a9..00000000 --- a/.github/workflows/windows-ci.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Windows CI (experimental) - -on: - pull_request: - push: - -jobs: - windows-smoke: - runs-on: windows-latest - continue-on-error: true #this is a smoke test (experimental); it won't block merges - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: pip - cache-dependency-path: | - packaging/nominatim-db/pyproject.toml - packaging/nominatim-api/pyproject.toml - - - name: Install latest flake8 - run: python -m pip install -U pip flake8 - - - name: Python linting - run: python -m flake8 src test/python test/bdd - - - name: Compile Python sources (smoke) - run: python -m compileall src/nominatim_db src/nominatim_api nominatim-cli.py