Skip to content

Commit

Permalink
reduce number of spatial queries
Browse files Browse the repository at this point in the history
  • Loading branch information
@michmuel
michmuel committed Aug 29, 2024
1 parent ec86cb5 commit 19e2f11
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 117 deletions.
2 changes: 2 additions & 0 deletions pyramid_oereb/contrib/data_sources/interlis_2_3/models/theme.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from sqlalchemy import LargeBinary, String, Integer, Date, Text
from geoalchemy2.types import Geometry as GeoAlchemyGeometry
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.orm import query_expression


class Models(object):
Expand Down Expand Up @@ -350,6 +351,7 @@ class Geometry(Base):
PublicLawRestriction,
backref='geometries'
)
inside_real_estate = query_expression()

class MultilingualUri(Base):
"""
Expand Down
136 changes: 79 additions & 57 deletions pyramid_oereb/contrib/data_sources/interlis_2_3/sources/plr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from shapely.geometry import Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, \
GeometryCollection
from sqlalchemy import or_
from sqlalchemy.orm import with_expression
from sqlalchemy.orm import selectinload
from geoalchemy2.functions import ST_DWithin

Expand Down Expand Up @@ -412,44 +413,87 @@ def get_document_records(self, params, public_law_restriction_from_db):
document_records = self.from_db_to_document_records(documents_from_db)
return document_records

def collect_related_geometries_by_real_estate(self, session, real_estate):
def collect_related_geometries_by_real_estate_and_bbox(self, session, real_estate, bbox):
"""
Extracts all geometries in the topic which have spatial relation with the passed real estate
Args:
session (sqlalchemy.orm.Session): The requested clean session instance ready for use
real_estate (pyramid_oereb.lib.records.real_estate.RealEstateRecord): The real
estate in its record representation.
bbox (shapely.geometry.base.BaseGeometry): The bbox to search the records.
Returns:
list: The result of the related geometries unique by the public law restriction id
"""
if self._tolerances is None:
query = session.query(self._model_).filter(
query = session.query(
self._model_
).options(
with_expression(
self._model_.inside_real_estate,
or_(
self._model_.point.ST_Intersects(
from_shape(real_estate.limit, srid=Config.get('srid'))
),
self._model_.line.ST_Intersects(
from_shape(real_estate.limit, srid=Config.get('srid'))
),
self._model_.surface.ST_Intersects(
from_shape(real_estate.limit, srid=Config.get('srid'))
)
)
)
).filter(
or_(
self._model_.point.ST_Intersects(from_shape(real_estate.limit, srid=Config.get('srid'))),
self._model_.line.ST_Intersects(from_shape(real_estate.limit, srid=Config.get('srid'))),
self._model_.surface.ST_Intersects(from_shape(real_estate.limit, srid=Config.get('srid')))
))
self._model_.point.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.line.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.surface.ST_Intersects(from_shape(bbox, srid=Config.get('srid')))
)
)
else:
query = session.query(self._model_).filter(
query = session.query(
self._model_
).options(
with_expression(
self._model_.inside_real_estate,
or_(
ST_DWithin(
self._model_.point,
from_shape(real_estate.limit, srid=Config.get('srid')),
self._tolerances['ALL']
),
ST_DWithin(
self._model_.line,
from_shape(real_estate.limit, srid=Config.get('srid')),
self._tolerances['ALL']
),
ST_DWithin(
self._model_.surface,
from_shape(real_estate.limit, srid=Config.get('srid')),
self._tolerances['ALL']
)
)
)
).filter(
or_(
ST_DWithin(
self._model_.point,
from_shape(real_estate.limit, srid=Config.get('srid')),
from_shape(bbox, srid=Config.get('srid')),
self._tolerances.get('ALL', self._tolerances.get('Point', 0))
),
ST_DWithin(
self._model_.line,
from_shape(real_estate.limit, srid=Config.get('srid')),
from_shape(bbox, srid=Config.get('srid')),
self._tolerances.get('ALL', self._tolerances.get('LineString', 0))
),
ST_DWithin(
self._model_.surface,
from_shape(real_estate.limit, srid=Config.get('srid')),
from_shape(bbox, srid=Config.get('srid')),
self._tolerances.get('ALL', self._tolerances.get('Polygon', 0))
)
))
)
)
return query.distinct(self._model_.public_law_restriction_id).options(
selectinload(self.models.Geometry.public_law_restriction)
.selectinload(self.models.PublicLawRestriction.geometries),
Expand All @@ -470,37 +514,6 @@ def collect_related_geometries_by_real_estate(self, session, real_estate):
.selectinload(self.models.MultilingualUri.localised_uri)
).all()

def collect_legend_entries_by_bbox(self, session, bbox, law_status):
"""
Extracts all legend entries in the topic which have spatial relation with the passed bounding box of
visible extent.
Args:
session (sqlalchemy.orm.Session): The requested clean session instance ready for use
bbox (shapely.geometry.base.BaseGeometry): The bbox to search the records.
law_status (str): String of the law status for which the legend entries should be queried.
Returns:
list: The result of the related geometries unique by the public law restriction id and law status
"""
distinct_legend_entry_ids = []
geometries = session.query(self._model_).filter(
or_(
self._model_.point.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.line.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.surface.ST_Intersects(from_shape(bbox, srid=Config.get('srid')))
)).distinct(self._model_.public_law_restriction_id).options(
selectinload(self.models.Geometry.public_law_restriction)
).all()

for geometry in geometries:
if geometry.public_law_restriction.legend_entry_id not in distinct_legend_entry_ids\
and geometry.public_law_restriction.law_status == law_status:
distinct_legend_entry_ids.append(geometry.public_law_restriction.legend_entry_id)

return session.query(self.legend_entry_model).filter(
self.legend_entry_model.t_id.in_((distinct_legend_entry_ids))).all()

def read(self, params, real_estate, bbox):
"""
The read point which creates a extract, depending on a passed real estate.
Expand All @@ -525,10 +538,10 @@ def read(self, params, real_estate, bbox):
# We need to investigate more in detail

# Try to find geometries which have spatial relation with real estate
geometry_results = self.collect_related_geometries_by_real_estate(
session, real_estate
geometry_results = self.collect_related_geometries_by_real_estate_and_bbox(
session, real_estate, bbox
)
if len(geometry_results) == 0:
if any([x.inside_real_estate for x in geometry_results]):
# We checked if there are spatially related elements in database. But there is none.
# So we can stop here.
self.records = [EmptyPlrRecord(
Expand All @@ -537,20 +550,29 @@ def read(self, params, real_estate, bbox):
else:
# We found spatially related elements. This means we need to extract the actual plr
# information related to the found geometries.
law_status_of_geometry = []
# get distinct values of law_status for all geometries found
for geometry in geometry_results:
if (geometry.public_law_restriction.law_status not in law_status_of_geometry):
law_status_of_geometry.append(geometry.public_law_restriction.law_status)

legend_entries_from_db = []
# get legend_entries per law_status
for law_status in law_status_of_geometry:
legend_entry_with_law_status = [
self.collect_legend_entries_by_bbox(session, bbox, law_status),
law_status
]
legend_entries_from_db.append(legend_entry_with_law_status)
legend_entries_from_db = []
for law_status in list(set([x.public_law_restriction.law_status
for x in geometry_results])):
legend_entries_from_db.append(
[
session.query(
self.legend_entry_model
).filter(
self.legend_entry_model.t_id.in_(
list(
set([x.public_law_restriction.legend_entry_id
for x in geometry_results
if x.public_law_restriction.law_status == law_status]))
)
).all(),
law_status
]
)

# keep only the geometries that intersects with the real estate
geometry_results = [x for x in geometry_results if x.inside_real_estate is True]

self.records = []
for geometry_result in geometry_results:
Expand Down
5 changes: 3 additions & 2 deletions pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,19 +107,20 @@ def document_records_from_oereblex(self, params, geolink, law_status, oereblex_p
self._queried_geolinks[identifier] = self._oereblex_source.records
return self._queried_geolinks[identifier]

def collect_related_geometries_by_real_estate(self, session, real_estate):
def collect_related_geometries_by_real_estate_and_bbox(self, session, real_estate, bbox):
"""
Extracts all geometries in the topic which have spatial relation with the passed real estate
Args:
session (sqlalchemy.orm.Session): The requested clean session instance ready for use
real_estate (pyramid_oereb.lib.records.real_estate.RealEstateRecord): The real
estate in its record representation.
bbox (shapely.geometry.base.BaseGeometry): The bbox to search the records.
Returns:
list: The result of the related geometries unique by the public law restriction id
"""
return self.handle_collection(session, real_estate.limit).distinct(
return self.handle_collection(session, real_estate.limit, bbox).distinct(
self._model_.public_law_restriction_id
).options(
selectinload(self.models.Geometry.public_law_restriction)
Expand Down
2 changes: 2 additions & 0 deletions pyramid_oereb/contrib/data_sources/standard/models/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from geoalchemy2.types import Geometry as GeoAlchemyGeometry
from sqlalchemy.orm import relationship
from sqlalchemy_utils import JSONType
from sqlalchemy.orm import query_expression


NAMING_CONVENTION = {
Expand Down Expand Up @@ -371,6 +372,7 @@ class Geometry(base):
PublicLawRestriction,
backref='geometries'
)
inside_real_estate = query_expression()

return Geometry

Expand Down
Loading

0 comments on commit 19e2f11

Please sign in to comment.