Merge pull request #3101 from lonvia/custom-geometry-type

Improve use of SQLAlchemy statement cache with search queries

.github/actions/build-nominatim/action.yml

@@ -27,10 +27,10 @@ runs:
           run: |
             sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson
             if [ "$FLAVOUR" == "oldstuff" ]; then
-                pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg
+                pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 datrie asyncpg
             else
                 sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
-                pip3 install sqlalchemy GeoAlchemy2 psycopg
+                pip3 install sqlalchemy psycopg
             fi
           shell: bash
           env:

docs/admin/Installation.md

@@ -49,7 +49,6 @@ For running Nominatim:
   * [psutil](https://github.com/giampaolo/psutil)
   * [Jinja2](https://palletsprojects.com/p/jinja/)
   * [SQLAlchemy](https://www.sqlalchemy.org/) (1.4+ with greenlet support)
-  * [GeoAlchemy2](https://geoalchemy-2.readthedocs.io/) (0.10+)
   * [asyncpg](https://magicstack.github.io/asyncpg) (0.8+)
   * [PyICU](https://pypi.org/project/PyICU/)
   * [PyYaml](https://pyyaml.org/) (5.1+)

lib-sql/functions/ranking.sql

@@ -284,3 +284,26 @@ BEGIN
 END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
+
+
+CREATE OR REPLACE FUNCTION weigh_search(search_vector INT[],
+                                        term_vectors TEXT[],
+                                        weight_vectors FLOAT[],
+                                        def_weight FLOAT)
+  RETURNS FLOAT
+  AS $$
+DECLARE
+  pos INT := 1;
+  terms TEXT;
+BEGIN
+  FOREACH terms IN ARRAY term_vectors
+  LOOP
+    IF search_vector @> terms::INTEGER[] THEN
+      RETURN weight_vectors[pos];
+    END IF;
+    pos := pos + 1;
+  END LOOP;
+  RETURN def_weight;
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;

nominatim/api/connection.py

@@ -10,11 +10,11 @@
 from typing import cast, Any, Mapping, Sequence, Union, Dict, Optional, Set
 
 import sqlalchemy as sa
-from geoalchemy2 import Geometry
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from nominatim.typing import SaFromClause
 from nominatim.db.sqlalchemy_schema import SearchTables
+from nominatim.db.sqlalchemy_types import Geometry
 from nominatim.api.logging import log
 
 class SearchConnection:
@@ -38,7 +38,7 @@ async def scalar(self, sql: sa.sql.base.Executable,
                     ) -> Any:
         """ Execute a 'scalar()' query on the connection.
         """
-        log().sql(self.connection, sql)
+        log().sql(self.connection, sql, params)
         return await self.connection.scalar(sql, params)
 
 
@@ -47,7 +47,7 @@ async def execute(self, sql: 'sa.Executable',
                      ) -> 'sa.Result[Any]':
         """ Execute a 'execute()' query on the connection.
         """
-        log().sql(self.connection, sql)
+        log().sql(self.connection, sql, params)
         return await self.connection.execute(sql, params)
 
 
@@ -112,4 +112,4 @@ async def get_class_table(self, cls: str, typ: str) -> Optional[SaFromClause]:
 
         return sa.Table(tablename, self.t.meta,
                         sa.Column('place_id', sa.BigInteger),
-                        sa.Column('centroid', Geometry(srid=4326, spatial_index=False)))
+                        sa.Column('centroid', Geometry))

nominatim/api/core.py

@@ -66,7 +66,8 @@ async def setup_database(self) -> None:
                        username=dsn.get('user'), password=dsn.get('password'),
                        host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None,
                        query=query)
-            engine = sa_asyncio.create_async_engine(dburl, future=True)
+            engine = sa_asyncio.create_async_engine(dburl, future=True,
+                                                    echo=self.config.get_bool('DEBUG_SQL'))
 
             try:
                 async with engine.begin() as conn:

nominatim/api/logging.py

@@ -7,11 +7,12 @@
 """
 Functions for specialised logging with HTML output.
 """
-from typing import Any, Iterator, Optional, List, Tuple, cast
+from typing import Any, Iterator, Optional, List, Tuple, cast, Union, Mapping, Sequence
 from contextvars import ContextVar
 import datetime as dt
 import textwrap
 import io
+import re
 
 import sqlalchemy as sa
 from sqlalchemy.ext.asyncio import AsyncConnection
@@ -74,22 +75,39 @@ def result_dump(self, heading: str, results: Iterator[Tuple[Any, Any]]) -> None:
         """
 
 
-    def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
+    def sql(self, conn: AsyncConnection, statement: 'sa.Executable',
+            params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None]) -> None:
         """ Print the SQL for the given statement.
         """
 
-    def format_sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> str:
+    def format_sql(self, conn: AsyncConnection, statement: 'sa.Executable',
+                   extra_params: Union[Mapping[str, Any],
+                                 Sequence[Mapping[str, Any]], None]) -> str:
         """ Return the comiled version of the statement.
         """
-        try:
-            return str(cast('sa.ClauseElement', statement)
-                         .compile(conn.sync_engine, compile_kwargs={"literal_binds": True}))
-        except sa.exc.CompileError:
-            pass
-        except NotImplementedError:
-            pass
+        compiled = cast('sa.ClauseElement', statement).compile(conn.sync_engine)
 
-        return str(cast('sa.ClauseElement', statement).compile(conn.sync_engine))
+        params = dict(compiled.params)
+        if isinstance(extra_params, Mapping):
+            for k, v in extra_params.items():
+                params[k] = str(v)
+        elif isinstance(extra_params, Sequence) and extra_params:
+            for k in extra_params[0]:
+                params[k] = f':{k}'
+
+        sqlstr = str(compiled)
+
+        if sa.__version__.startswith('1'):
+            try:
+                return sqlstr % tuple((repr(params.get(name, None))
+                                      for name in compiled.positiontup)) # type: ignore
+            except TypeError:
+                return sqlstr
+
+        # Fixes an odd issue with Python 3.7 where percentages are not
+        # quoted correctly.
+        sqlstr = re.sub(r'%(?!\()', '%%', sqlstr)
+        return sqlstr % params
 
 
 class HTMLLogger(BaseLogger):
@@ -183,9 +201,10 @@ def format_osm(osm_object: Optional[Tuple[str, int]]) -> str:
         self._write(f'</dl><b>TOTAL:</b> {total}</p>')
 
 
-    def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
+    def sql(self, conn: AsyncConnection, statement: 'sa.Executable',
+            params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None]) -> None:
         self._timestamp()
-        sqlstr = self.format_sql(conn, statement)
+        sqlstr = self.format_sql(conn, statement, params)
         if CODE_HIGHLIGHT:
             sqlstr = highlight(sqlstr, PostgresLexer(),
                                HtmlFormatter(nowrap=True, lineseparator='<br />'))
@@ -276,8 +295,9 @@ def result_dump(self, heading: str, results: Iterator[Tuple[Any, Any]]) -> None:
         self._write(f'TOTAL: {total}\n\n')
 
 
-    def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None:
-        sqlstr = '\n| '.join(textwrap.wrap(self.format_sql(conn, statement), width=78))
+    def sql(self, conn: AsyncConnection, statement: 'sa.Executable',
+            params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None]) -> None:
+        sqlstr = '\n| '.join(textwrap.wrap(self.format_sql(conn, statement, params), width=78))
         self._write(f"| {sqlstr}\n\n")
 
 

nominatim/api/results.py

@@ -263,7 +263,7 @@ def create_from_placex_row(row: Optional[SaRow],
                       rank_search=row.rank_search,
                       importance=row.importance,
                       country_code=row.country_code,
-                      centroid=Point.from_wkb(row.centroid.data),
+                      centroid=Point.from_wkb(row.centroid),
                       geometry=_filter_geometries(row))
 
 
@@ -288,7 +288,7 @@ def create_from_osmline_row(row: Optional[SaRow],
                      address=row.address,
                      postcode=row.postcode,
                      country_code=row.country_code,
-                     centroid=Point.from_wkb(row.centroid.data),
+                     centroid=Point.from_wkb(row.centroid),
                      geometry=_filter_geometries(row))
 
     if hnr is None:
@@ -321,7 +321,7 @@ def create_from_tiger_row(row: Optional[SaRow],
                      category=('place', 'houses' if hnr is None else 'house'),
                      postcode=row.postcode,
                      country_code='us',
-                     centroid=Point.from_wkb(row.centroid.data),
+                     centroid=Point.from_wkb(row.centroid),
                      geometry=_filter_geometries(row))
 
     if hnr is None:
@@ -350,7 +350,7 @@ def create_from_postcode_row(row: Optional[SaRow],
                       rank_search=row.rank_search,
                       rank_address=row.rank_address,
                       country_code=row.country_code,
-                      centroid=Point.from_wkb(row.centroid.data),
+                      centroid=Point.from_wkb(row.centroid),
                       geometry=_filter_geometries(row))
 
 
@@ -365,7 +365,7 @@ def create_from_country_row(row: Optional[SaRow],
 
     return class_type(source_table=SourceTable.COUNTRY,
                       category=('place', 'country'),
-                      centroid=Point.from_wkb(row.centroid.data),
+                      centroid=Point.from_wkb(row.centroid),
                       names=row.name,
                       rank_address=4, rank_search=4,
                       country_code=row.country_code)