feat: add MV to aggregate observations by meshes

Resolve #518

atlas/atlasAPI.py

@@ -45,31 +45,35 @@ def getObservationsMailleAndPointAPI(cd_ref):
         :returns: dict ({'point:<GeoJson>', 'maille': 'GeoJson})
         """
         session = db.session
+        points = vmObservationsRepository.searchObservationsChilds(session, cd_ref)
+        session.close()
+
+        connection = utils.engine.connect()
+        meshes = vmObservationsMaillesRepository.getObservationsByMeshes(session, cd_ref)
+        connection.close()
+
         observations = {
-            "point": vmObservationsRepository.searchObservationsChilds(session, cd_ref),
-            "maille": vmObservationsMaillesRepository.getObservationsMaillesChilds(
-                session, cd_ref
-            ),
+            "point": points,
+            "maille": meshes,
         }
-        session.close()
         return jsonify(observations)
 
 
 @api.route("/observationsMaille/<int:cd_ref>", methods=["GET"])
-def getObservationsMailleAPI(cd_ref, year_min=None, year_max=None):
+def getObservationsMailleAPI(cd_ref):
     """
     Retourne les observations d'un taxon par maille (et le nombre d'observation par maille)
 
     :returns: GeoJson
     """
-    session = db.session
-    observations = vmObservationsMaillesRepository.getObservationsMaillesChilds(
-        session,
+    connection = utils.engine.connect()
+    observations = vmObservationsMaillesRepository.getObservationsByMeshes(
+        connection,
         cd_ref,
         year_min=request.args.get("year_min"),
         year_max=request.args.get("year_max"),
     )
-    session.close()
+    connection.close()
     return jsonify(observations)
 
 
@@ -93,18 +97,20 @@ def getObservationsGenericApi(cd_ref: int):
     Returns:
         [type]: [description]
     """
-    session = db.session
-    observations = (
-        vmObservationsMaillesRepository.getObservationsMaillesChilds(
-            session,
+    if current_app.config["AFFICHAGE_MAILLE"]:
+        connection = utils.engine.connect()
+        observations = vmObservationsMaillesRepository.getObservationsByMeshes(
+            connection,
             cd_ref,
             year_min=request.args.get("year_min"),
             year_max=request.args.get("year_max"),
         )
-        if current_app.config["AFFICHAGE_MAILLE"]
-        else vmObservationsRepository.searchObservationsChilds(session, cd_ref)
-    )
-    session.close()
+        connection.close()
+    else:
+        session = utils.loadSession()
+        observations = vmObservationsRepository.searchObservationsChilds(session, cd_ref)
+        session.close()
+
     return jsonify(observations)
 
 

atlas/modeles/repositories/vmObservationsMaillesRepository.py

@@ -7,39 +7,47 @@
 from atlas.modeles.utils import deleteAccent, findPath
 
 
-def getObservationsMaillesChilds(session, cd_ref, year_min=None, year_max=None):
-    """
-    Retourne les mailles et le nombre d'observation par maille pour un taxon et ses enfants
-    sous forme d'un geojson
+def getObservationsByMeshes(connection, cd_ref, year_min=None, year_max=None):
+    sql = "SELECT * FROM atlas.find_all_taxons_childs(:cdRef) AS taxon_childs(cd_nom)"
+    results = connection.execute(text(sql), cdRef=cd_ref)
+    taxons_ids = [cd_ref]
+    for r in results:
+        taxons_ids.append(r.cd_nom)
+
+    year_clause = ""
+    if year_min and year_max:
+        year_clause = "AND voma.year >= :yearMin AND voma.year <= :yearMax "
+
+    sql = f"""
+        SELECT
+            mt.id_maille AS id_mesh,
+            mt.geojson_maille AS mesh_geojson,
+            MAX(voma."year")::int AS last_obs_year,
+            SUM(voma.nbr)::int AS obs_nbr
+        FROM atlas.vm_observations_meshes_agg AS voma
+            JOIN atlas.t_mailles_territoire AS mt
+                ON voma.mesh_id = mt.id_maille
+        WHERE voma.cd_ref = ANY(:cdRefList)
+            {year_clause}
+        GROUP BY mt.id_maille, mt.geojson_maille ;
     """
-    subquery = session.query(func.atlas.find_all_taxons_childs(cd_ref))
-    query = (
-        session.query(
-            func.count(VmObservationsMailles.id_observation).label("nb_obs"),
-            func.max(VmObservationsMailles.annee).label("last_observation"),
-            VmObservationsMailles.id_maille,
-            VmObservationsMailles.geojson_maille,
-        )
-        .group_by(VmObservationsMailles.id_maille, VmObservationsMailles.geojson_maille)
-        .filter(
-            or_(VmObservationsMailles.cd_ref.in_(subquery), VmObservationsMailles.cd_ref == cd_ref)
-        )
+
+    observations = connection.execute(
+        text(sql), cdRefList=taxons_ids, yearMin=year_min, yearMax=year_max
     )
-    if year_min and year_max:
-        query = query.filter(VmObservationsMailles.annee.between(year_min, year_max))
 
     return FeatureCollection(
         [
             Feature(
-                id=o.id_maille,
-                geometry=json.loads(o.geojson_maille),
+                id=o.id_mesh,
+                geometry=json.loads(o.mesh_geojson),
                 properties={
-                    "id_maille": o.id_maille,
-                    "nb_observations": o.nb_obs,
-                    "last_observation": o.last_observation,
+                    "id_maille": o.id_mesh,
+                    "nb_observations": o.obs_nbr,
+                    "last_observation": o.last_obs_year,
                 },
             )
-            for o in query.all()
+            for o in observations
         ]
     )
 

data/atlas/14.atlas.vm_observations_meshes_agg.sql

@@ -0,0 +1,15 @@
+CREATE MATERIALIZED VIEW atlas.vm_observations_meshes_agg AS
+    SELECT obs.cd_ref,
+        obs.id_maille AS id_mesh,
+        obs.annee AS "year",
+        COUNT(obs.id_observation) AS nbr
+    FROM atlas.vm_observations_mailles AS obs
+    GROUP BY obs.cd_ref, obs.id_maille, obs.annee
+    ORDER BY obs.cd_ref, obs.annee
+WITH DATA;
+
+-- View indexes:
+CREATE INDEX idx_voma_annee ON atlas.vm_observations_meshes_agg
+    USING btree ("year");
+CREATE INDEX idx_voma_id_maille_cd_ref ON atlas.vm_observations_meshes_agg
+    USING btree (id_mesh, cd_ref);

data/atlas/14.grant.sql → data/atlas/15.grant.sql

@@ -38,3 +38,4 @@ GRANT EXECUTE ON FUNCTION atlas.find_all_taxons_childs(integer) TO my_reader_use
 GRANT SELECT ON TABLE atlas.bib_taxref_rangs TO my_reader_user;
 GRANT SELECT ON TABLE atlas.t_mailles_territoire TO my_reader_user;
 GRANT SELECT ON TABLE atlas.vm_cor_taxon_organism TO my_reader_user;
+GRANT SELECT ON TABLE atlas.vm_observations_meshes_agg TO my_reader_user;

install_db.sh

@@ -426,6 +426,11 @@ if ! database_exists $db_name
         export PGPASSWORD=$owner_atlas_pass;psql -d $db_name -U $owner_atlas -h $db_host -p $db_port -f /tmp/atlas/13.atlas.vm_observations_mailles.sql  &>> log/install_db.log
         echo "[$(date +'%H:%M:%S')] Passed - Duration : $((($SECONDS-$time_temp)/60))m$((($SECONDS-$time_temp)%60))s"
 
+        echo "[$(date +'%H:%M:%S')] Creating atlas.vm_observations_meshes_agg..."
+        time_temp=$SECONDS
+        export PGPASSWORD=$owner_atlas_pass;psql -d $db_name -U $owner_atlas -h $db_host -p $db_port -f /tmp/atlas/14.atlas.vm_observations_meshes_agg.sql  &>> log/install_db.log
+        echo "[$(date +'%H:%M:%S')] Passed - Duration : $((($SECONDS-$time_temp)/60))m$((($SECONDS-$time_temp)%60))s"
+
         sudo -u postgres -s psql -d $db_name -c "ALTER TABLE atlas.bib_taxref_rangs OWNER TO "$owner_atlas";"
         sudo -u postgres -s psql -d $db_name -c "ALTER TABLE atlas.bib_taxref_rangs OWNER TO "$owner_atlas";"
         sudo -u postgres -s psql -d $db_name -c "ALTER FUNCTION atlas.create_vm_altitudes() OWNER TO "$owner_atlas";"
@@ -438,8 +443,8 @@ if ! database_exists $db_name
         # FR: Affectation de droits en lecture sur les VM à l'utilisateur de l'application ($user_pg)
         # EN: Assign read rights on VMs to the application user ($user_pg)
         echo "Grant..."
-        sudo sed -i "s/my_reader_user;$/$user_pg;/" /tmp/atlas/14.grant.sql
-        sudo -n -u postgres -s psql -d $db_name -f /tmp/atlas/14.grant.sql &>> log/install_db.log
+        sudo sed -i "s/my_reader_user;$/$user_pg;/" /tmp/atlas/15.grant.sql
+        sudo -n -u postgres -s psql -d $db_name -f /tmp/atlas/15.grant.sql &>> log/install_db.log
 
         # Clean file
         echo "Cleaning files..."