From a2a44b875a02db284d0f67eb9a20506622da2257 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 8 Aug 2024 11:53:27 +0200 Subject: [PATCH 1/4] add deprecation warnings throughout docs --- docs/admin/Deployment-PHP.md | 3 +++ docs/admin/Import.md | 3 +++ docs/admin/Installation.md | 3 ++- docs/admin/Migration.md | 15 ++++++++++----- docs/customize/SQLite.md | 5 +++++ docs/customize/Tokenizers.md | 5 +++++ 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/docs/admin/Deployment-PHP.md b/docs/admin/Deployment-PHP.md index 3ff86dad47..9416c53eef 100644 --- a/docs/admin/Deployment-PHP.md +++ b/docs/admin/Deployment-PHP.md @@ -1,5 +1,8 @@ # Deploying Nominatim using the PHP frontend +!!! danger + The PHP frontend is deprecated and will be removed in Nominatim 5.0. + The Nominatim API is implemented as a PHP application. The `website/` directory in the project directory contains the configured website. You can serve this in a production environment with any web server that is capable to run diff --git a/docs/admin/Import.md b/docs/admin/Import.md index 1a87ebbd85..5a365b236b 100644 --- a/docs/admin/Import.md +++ b/docs/admin/Import.md @@ -302,6 +302,9 @@ To run Nominatim via webservers like Apache or nginx, please continue reading #### Testing the PHP frontend +!!! danger + The PHP fronted is deprecated and will be removed in Nominatim 5.0. + You can run a small test server with the PHP frontend like this: ```sh diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index cd561718ae..e67371bd9d 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -72,7 +72,7 @@ For running the Python frontend: * [starlette](https://www.starlette.io/) * [uvicorn](https://www.uvicorn.org/) -For running the legacy PHP frontend: +For running the legacy PHP frontend (deprecated, will be removed in Nominatim 5.0): * [PHP](https://php.net) (7.3+) * PHP-pgsql @@ -194,6 +194,7 @@ sudo make install cmake: `cmake -DBUILD_MODULE=on ../Nominatim`. To compile the module you need to have the server development headers for PostgreSQL installed. On Ubuntu/Debian run: `sudo apt install postgresql-server-dev-` + The legacy tokenizer is deprecated and will be removed in Nominatim 5.0 Nominatim installs itself into `/usr/local` per default. To choose a different diff --git a/docs/admin/Migration.md b/docs/admin/Migration.md index e4db38d150..75f891412e 100644 --- a/docs/admin/Migration.md +++ b/docs/admin/Migration.md @@ -1,12 +1,17 @@ # Database Migrations -Since version 3.7.0 Nominatim offers automatic migrations. Please follow +Nominatim offers automatic migrations since version 3.7. Please follow the following steps: -* stop any updates that are potentially running -* update Nominatim to the newer version -* go to your project directory and run `nominatim admin --migrate` -* (optionally) restart updates +* Stop any updates that are potentially running +* Update the backend: `pip install -U nominatim-db` +* Go to your project directory and run `nominatim admin --migrate` +* Update the frontend: `pip install -U nominatim-api` +* (optionally) Restart updates + +If you are still using CMake for the installation of Nominatim, then you +need to update the software in one step before migrating the database. +It is not recommended to do this while the machine is serving requests. Below you find additional migrations and hints about other structural and breaking changes. **Please read them before running the migration.** diff --git a/docs/customize/SQLite.md b/docs/customize/SQLite.md index 9614feabb8..bda8cd5cdb 100644 --- a/docs/customize/SQLite.md +++ b/docs/customize/SQLite.md @@ -12,11 +12,16 @@ To use a SQLite database, you need to install: * SQLite (>= 3.30) * Spatialite (> 5.0.0) +* aiosqlite On Ubuntu/Debian, you can run: sudo apt install sqlite3 libsqlite3-mod-spatialite libspatialite7 +Install the aiosqlite Python package in your virtual environment: + + /srv/nominatim-venv/bin/pip install aiosqlite + ## Creating a new SQLite database Nominatim cannot import directly into SQLite database. Instead you have to diff --git a/docs/customize/Tokenizers.md b/docs/customize/Tokenizers.md index ee51fec05b..49e86a5009 100644 --- a/docs/customize/Tokenizers.md +++ b/docs/customize/Tokenizers.md @@ -17,6 +17,11 @@ they can be configured. ## Legacy tokenizer +!!! danger + The Legacy tokenizer is deprecated and will be removed in Nominatim 5.0. + If you still use a database with the legacy tokenizer, you must reimport + it using the ICU tokenizer below. + The legacy tokenizer implements the analysis algorithms of older Nominatim versions. It uses a special Postgresql module to normalize names and queries. This tokenizer is automatically installed and used when upgrading an older From 3cc20581aeac2c263fe1e9f6d4c8ab3e428bea00 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 8 Aug 2024 22:19:11 +0200 Subject: [PATCH 2/4] osm2pgsq: do not use deprecated tablespace options --- src/nominatim_db/tools/exec_utils.py | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/nominatim_db/tools/exec_utils.py b/src/nominatim_db/tools/exec_utils.py index 406e2511e8..2b01b5b5e1 100644 --- a/src/nominatim_db/tools/exec_utils.py +++ b/src/nominatim_db/tools/exec_utils.py @@ -7,7 +7,7 @@ """ Helper functions for executing external programs. """ -from typing import Any, Mapping +from typing import Any, Mapping, List import logging import os import subprocess @@ -31,6 +31,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: env = get_pg_env(options['dsn']) cmd = [_find_osm2pgsql_cmd(options['osm2pgsql']), + '--append' if options['append'] else '--create', '--slim', '--log-progress', 'true', '--number-processes', '1' if options['append'] else str(options['threads']), @@ -42,25 +43,20 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'), os.environ.get('LUAPATH', ';'))) cmd.extend(('--output', 'flex')) + + for flavour in ('data', 'index'): + if options['tablespaces'][f"main_{flavour}"]: + env[f"NOMINATIM_TABLESPACE_PLACE_{flavour.upper()}"] = \ + options['tablespaces'][f"main_{flavour}"] else: cmd.extend(('--output', 'gazetteer', '--hstore', '--latlon')) + cmd.extend(_mk_tablespace_options('main', options)) - cmd.append('--append' if options['append'] else '--create') if options['flatnode_file']: cmd.extend(('--flat-nodes', options['flatnode_file'])) - for key, param in (('slim_data', '--tablespace-slim-data'), - ('slim_index', '--tablespace-slim-index'), - ('main_data', '--tablespace-main-data'), - ('main_index', '--tablespace-main-index')): - if options['tablespaces'][key]: - cmd.extend((param, options['tablespaces'][key])) - - if options['tablespaces']['main_data']: - env['NOMINATIM_TABLESPACE_PLACE_DATA'] = options['tablespaces']['main_data'] - if options['tablespaces']['main_index']: - env['NOMINATIM_TABLESPACE_PLACE_INDEX'] = options['tablespaces']['main_index'] + cmd.extend(_mk_tablespace_options('slim', options)) if options.get('disable_jit', False): env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0' @@ -78,6 +74,16 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: env=env, check=True) +def _mk_tablespace_options(ttype: str, options: Mapping[str, Any]) -> List[str]: + cmds: List[str] = [] + for flavour in ('data', 'index'): + if options['tablespaces'][f"{ttype}_{flavour}"]: + cmds.extend((f"--tablespace-{ttype}-{flavour}", + options['tablespaces'][f"{ttype}_{flavour}"])) + + return cmds + + def _find_osm2pgsql_cmd(cmdline: str) -> str: if cmdline is not None: return cmdline From 6527b7cdcdc22eb8bab8851893a4699b60805a2b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 9 Aug 2024 14:44:49 +0200 Subject: [PATCH 3/4] fail if osm2pgsql is not recent enough --- src/nominatim_db/tools/exec_utils.py | 30 +++++++++++++++++++++++++--- src/nominatim_db/version.py | 1 + test/python/tools/conftest.py | 19 +++++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/nominatim_db/tools/exec_utils.py b/src/nominatim_db/tools/exec_utils.py index 2b01b5b5e1..1adcc777b0 100644 --- a/src/nominatim_db/tools/exec_utils.py +++ b/src/nominatim_db/tools/exec_utils.py @@ -7,14 +7,17 @@ """ Helper functions for executing external programs. """ -from typing import Any, Mapping, List +from typing import Any, Mapping, List, Optional import logging import os +import re import subprocess import shutil from ..typing import StrPath from ..db.connection import get_pg_env +from ..errors import UsageError +from ..version import OSM2PGSQL_REQUIRED_VERSION LOG = logging.getLogger() @@ -28,6 +31,8 @@ def run_php_server(server_address: str, base_dir: StrPath) -> None: def run_osm2pgsql(options: Mapping[str, Any]) -> None: """ Run osm2pgsql with the given options. """ + _check_osm2pgsql_version(options['osm2pgsql']) + env = get_pg_env(options['dsn']) cmd = [_find_osm2pgsql_cmd(options['osm2pgsql']), @@ -84,12 +89,31 @@ def _mk_tablespace_options(ttype: str, options: Mapping[str, Any]) -> List[str]: return cmds -def _find_osm2pgsql_cmd(cmdline: str) -> str: +def _find_osm2pgsql_cmd(cmdline: Optional[str]) -> str: if cmdline is not None: return cmdline in_path = shutil.which('osm2pgsql') if in_path is None: - raise RuntimeError('osm2pgsql executable not found. Please install osm2pgsql first.') + raise UsageError('osm2pgsql executable not found. Please install osm2pgsql first.') return str(in_path) + + +def _check_osm2pgsql_version(cmdline: Optional[str]) -> None: + cmd = [_find_osm2pgsql_cmd(cmdline), '--version'] + + result = subprocess.run(cmd, capture_output=True, check=True) + + if not result.stderr: + raise UsageError("osm2pgsql does not print version information.") + + verinfo = result.stderr.decode('UTF-8') + + match = re.search(r'osm2pgsql version (\d+)\.(\d+)', verinfo) + if match is None: + raise UsageError(f"No version information found in output: {verinfo}") + + if (int(match[1]), int(match[2])) < OSM2PGSQL_REQUIRED_VERSION: + raise UsageError(f"osm2pgsql is too old. Found version {match[1]}.{match[2]}. " + f"Need at least version {'.'.join(map(str, OSM2PGSQL_REQUIRED_VERSION))}.") diff --git a/src/nominatim_db/version.py b/src/nominatim_db/version.py index fceee5d04f..588a31c8c5 100644 --- a/src/nominatim_db/version.py +++ b/src/nominatim_db/version.py @@ -62,6 +62,7 @@ def parse_version(version: str) -> NominatimVersion: POSTGRESQL_REQUIRED_VERSION = (9, 6) POSTGIS_REQUIRED_VERSION = (2, 2) +OSM2PGSQL_REQUIRED_VERSION = (1, 8) # Cmake sets a variable @GIT_HASH@ by executing 'git --log'. It is not run # on every execution of 'make'. diff --git a/test/python/tools/conftest.py b/test/python/tools/conftest.py index 60b25c3b46..0098747e52 100644 --- a/test/python/tools/conftest.py +++ b/test/python/tools/conftest.py @@ -7,10 +7,23 @@ import pytest @pytest.fixture -def osm2pgsql_options(temp_db): - """ A standard set of options for osm2pgsql. +def osm2pgsql_options(temp_db, tmp_path): + """ A standard set of options for osm2pgsql + together with a osm2pgsql mock that just reflects the command line. """ - return dict(osm2pgsql='echo', + osm2pgsql_exec = tmp_path / 'osm2pgsql_mock' + + osm2pgsql_exec.write_text("""#!/bin/sh + +if [ "$*" = "--version" ]; then + >&2 echo "2024-08-09 11:16:23 osm2pgsql version 11.7.2 (11.7.2)" +else + echo "$@" +fi + """) + osm2pgsql_exec.chmod(0o777) + + return dict(osm2pgsql=str(osm2pgsql_exec), osm2pgsql_cache=10, osm2pgsql_style='style.file', threads=1, From fd33ef92dcd30de9fe1ba218a474899e6707ca0d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 9 Aug 2024 17:40:15 +0200 Subject: [PATCH 4/4] add deprecation warnings in the code --- CMakeLists.txt | 4 ++++ src/nominatim_db/cli.py | 10 +++++++++- src/nominatim_db/tokenizer/legacy_tokenizer.py | 8 +++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd5c3110e3..d1c2702a06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -271,3 +271,7 @@ if (INSTALL_MUNIN_PLUGINS) munin/nominatim_requests DESTINATION ${NOMINATIM_MUNINDIR}) endif() + +message(WARNING "Building with CMake is deprecated and will be removed in Nominatim 5.0." + "Use Nominatim pip packages instead.\n" + "See https://nominatim.org/release-docs/develop/admin/Installation/#downloading-and-building-nominatim") diff --git a/src/nominatim_db/cli.py b/src/nominatim_db/cli.py index 9327866882..88810df5a6 100644 --- a/src/nominatim_db/cli.py +++ b/src/nominatim_db/cli.py @@ -119,7 +119,13 @@ def run(self, **kwargs: Any) -> int: log.warning('Using project directory: %s', str(args.project_dir)) try: - return args.command.run(args) + ret = args.command.run(args) + + if args.config.TOKENIZER == 'legacy': + log.warning('WARNING: the "legacy" tokenizer is deprecated ' + 'and will be removed in Nominatim 5.0.') + + return ret except UsageError as exception: if log.isEnabledFor(logging.DEBUG): raise # use Python's exception printing @@ -169,6 +175,8 @@ def run(self, args: NominatimArgs) -> int: if args.engine == 'php': if args.config.lib_dir.php is None: raise UsageError("PHP frontend not configured.") + LOG.warning('\n\nWARNING: the PHP frontend is deprecated ' + 'and will be removed in Nominatim 5.0.\n\n') run_php_server(args.server, args.project_dir / 'website') else: asyncio.run(self.run_uvicorn(args)) diff --git a/src/nominatim_db/tokenizer/legacy_tokenizer.py b/src/nominatim_db/tokenizer/legacy_tokenizer.py index fa4b3b99ca..04b7b8814c 100644 --- a/src/nominatim_db/tokenizer/legacy_tokenizer.py +++ b/src/nominatim_db/tokenizer/legacy_tokenizer.py @@ -38,10 +38,12 @@ def create(dsn: str, data_dir: Path) -> 'LegacyTokenizer': """ Create a new instance of the tokenizer provided by this module. """ + LOG.warning('WARNING: the legacy tokenizer is deprecated ' + 'and will be removed in Nominatim 5.0.') return LegacyTokenizer(dsn, data_dir) -def _install_module(config_module_path: str, src_dir: Path, module_dir: Path) -> str: +def _install_module(config_module_path: str, src_dir: Optional[Path], module_dir: Path) -> str: """ Copies the PostgreSQL normalisation module into the project directory if necessary. For historical reasons the module is saved in the '/module' subdirectory and not with the other tokenizer @@ -55,6 +57,10 @@ def _install_module(config_module_path: str, src_dir: Path, module_dir: Path) -> LOG.info("Using custom path for database module at '%s'", config_module_path) return config_module_path + # Otherwise a source dir must be given. + if src_dir is None: + raise UsageError("The legacy tokenizer cannot be used with the Nominatim pip module.") + # Compatibility mode for builddir installations. if module_dir.exists() and src_dir.samefile(module_dir): LOG.info('Running from build directory. Leaving database module as is.')