From 158a8933ae08fbf649c10c18fac7f6886279d1e2 Mon Sep 17 00:00:00 2001 From: mandresy andri <70854158+mandresyandri@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:57:30 +0000 Subject: [PATCH 1/4] Update datas for gauges --- Dashboards/demo_streamlit.py | 64 +++++++------------------------- Dashboards/my_data/datasets.py | 16 +++++++- Dashboards/my_data/model.py | 13 ++++++- Dashboards/my_data/sql_stuff.sql | 38 +++++++++++++++++++ Dashboards/tests/test.py | 38 ++++++++++--------- 5 files changed, 97 insertions(+), 72 deletions(-) create mode 100644 Dashboards/my_data/sql_stuff.sql diff --git a/Dashboards/demo_streamlit.py b/Dashboards/demo_streamlit.py index 6a99117..8e9e4f0 100644 --- a/Dashboards/demo_streamlit.py +++ b/Dashboards/demo_streamlit.py @@ -7,7 +7,9 @@ # Source : https://discuss.streamlit.io/t/develop-a-dashboard-app-with-streamlit-using-plotly/37148/4 # run with : streamlit run Dashboards/demo_streamlit.py +# Finalement > prendre les données issue de la vue lichen_ecology = df.get_lichen_ecology() +lichen_frequency = df.get_lichen_frequency() # Debug data # st.dataframe(df) @@ -24,10 +26,14 @@ gauge = {'axis': {'range': [0, 100], 'dtick': 25}, 'bar': {'color': "#000000"}, 'steps' : [ - {'range': [0, 25], 'color': "#E3D7FF"}, - {'range': [25, 50], 'color': "#AFA2FF"}, - {'range': [50, 75], 'color': "#7A89C2"}, - {'range': [75, 100], 'color': "#72788D"} + # {'range': [0, 25], 'color': "#E3D7FF"}, + # {'range': [25, 50], 'color': "#AFA2FF"}, + # {'range': [50, 75], 'color': "#7A89C2"}, + # {'range': [75, 100], 'color': "#72788D"} + {'range': [0, 25], 'color': "green"}, + {'range': [25, 50], 'color': "yellow"}, + {'range': [50, 75], 'color': "orange"}, + {'range': [75, 100], 'color': "red"} ], 'threshold' : {'line': {'color': "#000000", 'width': 4}, 'thickness': 0.75, 'value': artificialisation_proportions["intermediate"]} })) @@ -60,53 +66,9 @@ hand_length = np.sqrt(2) / 4 hand_angle = np.pi * (1 - (max(min_value, min(max_value, current_value)) - min_value) / (max_value - min_value)) -## Version 3 non retenue du code python -# fig3 = go.Figure( -# data=[ -# go.Pie( -# values=[0.5] + (np.ones(n_quadrants) / 2 / n_quadrants).tolist(), -# rotation=90, -# hole=0.5, -# marker_colors=quadrant_colors, -# textinfo="text", -# hoverinfo="skip", -# ), -# ], -# layout=go.Layout( -# showlegend=False, -# margin=dict(b=0,t=10,l=10,r=10), -# width=450, -# height=450, -# paper_bgcolor=plot_bgcolor, -# annotations=[ -# go.layout.Annotation( -# text=f"Degrés d'artificialisation:
{current_value} %", -# x=0.5, xanchor="center", xref="paper", -# y=0.25, yanchor="bottom", yref="paper", -# showarrow=False, -# ) -# ], -# shapes=[ -# go.layout.Shape( -# type="circle", -# x0=0.48, x1=0.52, -# y0=0.48, y1=0.52, -# fillcolor="#333", -# line_color="#333", -# ), -# go.layout.Shape( -# type="line", -# x0=0.5, x1=0.5 + hand_length * np.cos(hand_angle), -# y0=0.5, y1=0.5 + hand_length * np.sin(hand_angle), -# line=dict(color="#333", width=4) -# ) -# ] -# ) -# ) - # Display streamlit st.title("Dataviz POC") -tab1, tab2, tab3= st.tabs(["Gauge", "Histogram", "df debug"]) +tab1, tab2, tab3= st.tabs(["Gauge", "Histogram", "df Fréquences"]) with tab1: st.write("**Mode de calcul**`df['poleotolerance'].value_counts(normalize=True) * 100`") st.plotly_chart(fig1) @@ -115,6 +77,6 @@ st.plotly_chart(fig2) with tab3: - st.write("Debug du dataset") - # st.dataframe(df) + st.write("les données de fréquences") + st.dataframe(lichen_frequency.groupby("id").sum()) \ No newline at end of file diff --git a/Dashboards/my_data/datasets.py b/Dashboards/my_data/datasets.py index 4b138bd..21fc8e7 100644 --- a/Dashboards/my_data/datasets.py +++ b/Dashboards/my_data/datasets.py @@ -1,6 +1,6 @@ import pandas as pd from my_data.db_connect import get_session -from my_data.model import Tree, TreeSpecies, Observation, Lichen, LichenSpecies, Environment, Table, LichenEcology +from my_data.model import Tree, TreeSpecies, Observation, Lichen, LichenSpecies, Environment, Table, LichenEcology, LichenFrequency session = get_session() @@ -113,4 +113,16 @@ def get_lichen_ecology(): "eutrophication": lichen_ecology.eutrophication, "poleotolerance" : lichen_ecology.poleotolerance }) - return pd.DataFrame(lichen_ecology_data) \ No newline at end of file + return pd.DataFrame(lichen_ecology_data) + +# Nouvelles données table fréquence +def get_lichen_frequency(): + data = session.query(LichenFrequency).all() + lichen_frequency_data = [] + for lichen_frequency in data: + lichen_frequency_data.append({ + "id": lichen_frequency.id_site, + "main_lichenspecies": lichen_frequency.main_lichenspecies, + "frequency": lichen_frequency.frequency + }) + return pd.DataFrame(lichen_frequency_data) diff --git a/Dashboards/my_data/model.py b/Dashboards/my_data/model.py index d7e13ff..1aef357 100644 --- a/Dashboards/my_data/model.py +++ b/Dashboards/my_data/model.py @@ -1,5 +1,6 @@ -from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, ARRAY, Boolean, Date, Text, Float +from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, ARRAY, Boolean, Date, Text, Float, text from sqlalchemy.orm import declarative_base, relationship +from my_data.db_connect import engine Base = declarative_base() @@ -97,4 +98,12 @@ class LichenEcology(Base): aridity = Column(String(255)) eutrophication = Column(String(255)) poleotolerance = Column(String(255)) - \ No newline at end of file + +# Table pour les fréquences +class LichenFrequency(Base): + __tablename__ = 'lichen_frequency' + __table_args__ = {'autoload_with': engine} + + id_site = Column(BigInteger, primary_key=True) + main_lichenspecies = Column(String(255)) + frequency = Column(BigInteger) \ No newline at end of file diff --git a/Dashboards/my_data/sql_stuff.sql b/Dashboards/my_data/sql_stuff.sql new file mode 100644 index 0000000..1931716 --- /dev/null +++ b/Dashboards/my_data/sql_stuff.sql @@ -0,0 +1,38 @@ +-- Listing des modifications faites sur la base de données +-- POC requête pour afficher les fréquences 17/07/2024 +SELECT + o.id AS id_site, + ls.name AS main_lichenspecies, + COUNT(l.id) AS frequency +FROM + main_observation o + JOIN main_tree t ON o.id = t.observation_id + JOIN main_lichen l ON o.id = l.observation_id + JOIN main_lichenspecies ls ON l.species_id = ls.id + +GROUP BY + o.id, ls.name + +ORDER BY o.id, frequency DESC; + +-- Pour la commandline PSQL +SELECT o.id AS id_site, ls.name AS main_lichenspecies, COUNT(l.id) AS frequency FROM main_observation o JOIN main_tree t ON o.id = t.observation_id JOIN main_lichen l ON o.id = l.observation_id JOIN main_lichenspecies ls ON l.species_id = ls.id GROUP BY o.id, ls.name ORDER BY o.id, frequency DESC; + +-- Créer une table view récupérable dans le modèle intégrée dans la base de données le 17/07/2024 +CREATE VIEW lichen_frequency AS +SELECT + o.id AS id_site, + ls.name AS main_lichenspecies, + COUNT(l.id) AS frequency +FROM + main_observation o + JOIN main_tree t ON o.id = t.observation_id + JOIN main_lichen l ON o.id = l.observation_id + JOIN main_lichenspecies ls ON l.species_id = ls.id +GROUP BY + o.id, ls.name +ORDER BY + o.id, frequency DESC; + +-- Pour la commandline PSQL +CREATE VIEW lichen_frequency AS SELECT o.id AS id_site, ls.name AS main_lichenspecies, COUNT(l.id) AS frequency FROM main_observation o JOIN main_tree t ON o.id = t.observation_id JOIN main_lichen l ON o.id = l.observation_id JOIN main_lichenspecies ls ON l.species_id = ls.id GROUP BY o.id, ls.name ORDER BY o.id, frequency DESC; diff --git a/Dashboards/tests/test.py b/Dashboards/tests/test.py index c709ec3..83eb3c9 100644 --- a/Dashboards/tests/test.py +++ b/Dashboards/tests/test.py @@ -6,7 +6,7 @@ chemin_dossier_parent = Path(__file__).parent.parent sys.path.append(str(chemin_dossier_parent)) from my_data.db_connect import get_session -from my_data.datasets import get_environment_data, get_lichen_data, get_lichen_species_data, get_observation_data, get_table_data, get_tree_data, get_tree_species, get_lichen_ecology +from my_data.datasets import get_environment_data, get_lichen_data, get_lichen_species_data, get_observation_data, get_table_data, get_tree_data, get_tree_species, get_lichen_ecology, get_lichen_frequency session = get_session() @@ -19,28 +19,32 @@ tree_df = get_tree_data() tree_species_df = get_tree_species() lichen_ecology_df = get_lichen_ecology() +lichen_frequency_df = get_lichen_frequency() # Affichage des datasets > test dataset -print("\nEnvironment Data") -print(environment_df.head()) +# print("\nEnvironment Data") +# print(environment_df.head()) -print("\nLichen Data") -print(lichen_df.head()) +# print("\nLichen Data") +# print(lichen_df.head()) -print("\nLichen Species Data") -print(lichen_species_df.head()) +# print("\nLichen Species Data") +# print(lichen_species_df.head()) -print("\nObservation Data") -print(observation_df.head()) +# print("\nObservation Data") +# print(observation_df.head()) -print("\nTable Data") -print(table_df.head()) +# print("\nTable Data") +# print(table_df.head()) -print("\nTree Data") -print(tree_df.head()) +# print("\nTree Data") +# print(tree_df.head()) -print("\nTree Species Data") -print(tree_species_df.head()) +# print("\nTree Species Data") +# print(tree_species_df.head()) -print("\nLichen Ecology Data") -print(lichen_ecology_df) +# print("\nLichen Ecology Data") +# print(lichen_ecology_df) + +# Correction de l'appel de méthode +print(lichen_frequency_df.head()) \ No newline at end of file From f342827acb3e4d07ef629b096af8c739e26323ac Mon Sep 17 00:00:00 2001 From: mandresy andri <70854158+mandresyandri@users.noreply.github.com> Date: Tue, 30 Jul 2024 06:05:16 +0000 Subject: [PATCH 2/4] Avancement groupby calcul --- Dashboards/demo_streamlit.py | 18 ++++++++++++++---- Dashboards/my_data/model.py | 2 +- Dashboards/tests/test.py | 1 - 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Dashboards/demo_streamlit.py b/Dashboards/demo_streamlit.py index 8e9e4f0..e23400f 100644 --- a/Dashboards/demo_streamlit.py +++ b/Dashboards/demo_streamlit.py @@ -11,8 +11,19 @@ lichen_ecology = df.get_lichen_ecology() lichen_frequency = df.get_lichen_frequency() -# Debug data -# st.dataframe(df) +st.write(lichen_ecology) + +# Calcul +def calc_frequences(df): + df_agg = df.groupby("main_lichenspecies").agg({ + "id": "first", + "frequency": "sum" + }).reset_index() + + return df_agg + +calc_freq = calc_frequences(lichen_frequency) + # Calculer les proportions des espèces selon leur tolérance à l'anthropisation artificialisation_proportions = lichen_ecology['poleotolerance'].value_counts(normalize=True) * 100 @@ -78,5 +89,4 @@ with tab3: st.write("les données de fréquences") - st.dataframe(lichen_frequency.groupby("id").sum()) - \ No newline at end of file + st.write(calc_freq) diff --git a/Dashboards/my_data/model.py b/Dashboards/my_data/model.py index 1aef357..1ba10b3 100644 --- a/Dashboards/my_data/model.py +++ b/Dashboards/my_data/model.py @@ -106,4 +106,4 @@ class LichenFrequency(Base): id_site = Column(BigInteger, primary_key=True) main_lichenspecies = Column(String(255)) - frequency = Column(BigInteger) \ No newline at end of file + frequency = Column(BigInteger) diff --git a/Dashboards/tests/test.py b/Dashboards/tests/test.py index 83eb3c9..ab0c0b6 100644 --- a/Dashboards/tests/test.py +++ b/Dashboards/tests/test.py @@ -47,4 +47,3 @@ # print(lichen_ecology_df) # Correction de l'appel de méthode -print(lichen_frequency_df.head()) \ No newline at end of file From 4143563fa153b58b752947918fa7c43b6fda91e2 Mon Sep 17 00:00:00 2001 From: mandresy andri <70854158+mandresyandri@users.noreply.github.com> Date: Wed, 31 Jul 2024 06:28:06 +0000 Subject: [PATCH 3/4] avancement calcul avec filtre artificialisation --- Dashboards/demo_streamlit.py | 44 +++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/Dashboards/demo_streamlit.py b/Dashboards/demo_streamlit.py index e23400f..bc14de5 100644 --- a/Dashboards/demo_streamlit.py +++ b/Dashboards/demo_streamlit.py @@ -11,9 +11,7 @@ lichen_ecology = df.get_lichen_ecology() lichen_frequency = df.get_lichen_frequency() -st.write(lichen_ecology) - -# Calcul +# Calcul somme des fréquences def calc_frequences(df): df_agg = df.groupby("main_lichenspecies").agg({ "id": "first", @@ -23,15 +21,45 @@ def calc_frequences(df): return df_agg calc_freq = calc_frequences(lichen_frequency) +calc_freq = calc_freq[["main_lichenspecies", "frequency"]] + +def deg_artif(id_site: int, species_name : str): + # Calcul filtrable + freq = lichen_frequency[lichen_frequency["id"] == id_site]["frequency"].values[0] + freq_g = calc_freq[calc_freq["main_lichenspecies"] == species_name]["frequency"].values[0] + + return round(freq / freq_g * 100, 2) + +# Sélection du site +id_site = st.selectbox( + "Sur quel site voulez-vous ?", + lichen_frequency["id"], + index=None, + placeholder="site n°", +) + +# Sélection des espèces +species_name = st.selectbox( + "Sur quel espèce voulez-vous ?", + calc_freq["main_lichenspecies"], + index=None, + placeholder="Je sélectionne l'espèce...", +) -# Calculer les proportions des espèces selon leur tolérance à l'anthropisation -artificialisation_proportions = lichen_ecology['poleotolerance'].value_counts(normalize=True) * 100 +# Affichage des éléments +if id_site and species_name != None: + pass +else: + id_site = 460 + species_name = "Physcia aipolia/stellaris" +# le calcul +artificialisation_proportions = deg_artif(id_site, species_name) # # Dataviz charts fig1 = go.Figure(go.Indicator( domain = {'x': [0, 1], 'y': [0, 1]}, - value = artificialisation_proportions["intermediate"], + value = artificialisation_proportions, mode = "gauge+number", title = {'text': "Degré d'artificialisation"}, gauge = {'axis': {'range': [0, 100], 'dtick': 25}, @@ -46,7 +74,7 @@ def calc_frequences(df): {'range': [50, 75], 'color': "orange"}, {'range': [75, 100], 'color': "red"} ], - 'threshold' : {'line': {'color': "#000000", 'width': 4}, 'thickness': 0.75, 'value': artificialisation_proportions["intermediate"]} + 'threshold' : {'line': {'color': "#000000", 'width': 4}, 'thickness': 0.75, 'value': artificialisation_proportions} })) x_values = [10, 8, 6, 5, 4, 3, 2, 1] @@ -81,7 +109,7 @@ def calc_frequences(df): st.title("Dataviz POC") tab1, tab2, tab3= st.tabs(["Gauge", "Histogram", "df Fréquences"]) with tab1: - st.write("**Mode de calcul**`df['poleotolerance'].value_counts(normalize=True) * 100`") + st.write(f"Degrés d'artificialisation sur le site n°**{id_site}** pour l'espèce **{species_name}**") st.plotly_chart(fig1) with tab2: From 5759006606ad1a6f04d48e9f3d671a9f2dc886b5 Mon Sep 17 00:00:00 2001 From: mandresy andri <70854158+mandresyandri@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:12:22 +0000 Subject: [PATCH 4/4] update data pour calcul --- Dashboards/my_data/datasets.py | 5 ++--- Dashboards/my_data/model.py | 5 +++-- Dashboards/my_data/sql_stuff.sql | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Dashboards/my_data/datasets.py b/Dashboards/my_data/datasets.py index 21fc8e7..cd92990 100644 --- a/Dashboards/my_data/datasets.py +++ b/Dashboards/my_data/datasets.py @@ -1,4 +1,5 @@ import pandas as pd +from sqlalchemy import text from my_data.db_connect import get_session from my_data.model import Tree, TreeSpecies, Observation, Lichen, LichenSpecies, Environment, Table, LichenEcology, LichenFrequency @@ -100,7 +101,6 @@ def get_tree_species(): }) return pd.DataFrame(tree_species_data) -# Nouvelles données def get_lichen_ecology(): data = session.query(LichenEcology).all() lichen_ecology_data = [] @@ -111,11 +111,10 @@ def get_lichen_ecology(): "pH": lichen_ecology.pH, "aridity": lichen_ecology.aridity, "eutrophication": lichen_ecology.eutrophication, - "poleotolerance" : lichen_ecology.poleotolerance + "poleotolerance": lichen_ecology.poleotolerance }) return pd.DataFrame(lichen_ecology_data) -# Nouvelles données table fréquence def get_lichen_frequency(): data = session.query(LichenFrequency).all() lichen_frequency_data = [] diff --git a/Dashboards/my_data/model.py b/Dashboards/my_data/model.py index 1ba10b3..7d72c4e 100644 --- a/Dashboards/my_data/model.py +++ b/Dashboards/my_data/model.py @@ -104,6 +104,7 @@ class LichenFrequency(Base): __tablename__ = 'lichen_frequency' __table_args__ = {'autoload_with': engine} - id_site = Column(BigInteger, primary_key=True) + id = Column(BigInteger, primary_key=True, autoincrement=True) + id_site = Column(BigInteger) main_lichenspecies = Column(String(255)) - frequency = Column(BigInteger) + frequency = Column(BigInteger) \ No newline at end of file diff --git a/Dashboards/my_data/sql_stuff.sql b/Dashboards/my_data/sql_stuff.sql index 1931716..75a5c95 100644 --- a/Dashboards/my_data/sql_stuff.sql +++ b/Dashboards/my_data/sql_stuff.sql @@ -36,3 +36,23 @@ ORDER BY -- Pour la commandline PSQL CREATE VIEW lichen_frequency AS SELECT o.id AS id_site, ls.name AS main_lichenspecies, COUNT(l.id) AS frequency FROM main_observation o JOIN main_tree t ON o.id = t.observation_id JOIN main_lichen l ON o.id = l.observation_id JOIN main_lichenspecies ls ON l.species_id = ls.id GROUP BY o.id, ls.name ORDER BY o.id, frequency DESC; + + +-- Update SQL command +CREATE VIEW lichen_frequency AS +SELECT + ROW_NUMBER() OVER (ORDER BY o.id, ls.name) AS id, + o.id AS id_site, + ls.name AS main_lichenspecies, + COUNT(l.id) AS frequency +FROM + main_observation o + JOIN main_tree t ON o.id = t.observation_id + JOIN main_lichen l ON o.id = l.observation_id + JOIN main_lichenspecies ls ON l.species_id = ls.id +GROUP BY + o.id, ls.name +ORDER BY + o.id, frequency DESC; + +CREATE VIEW lichen_frequency AS SELECT ROW_NUMBER() OVER (ORDER BY o.id, ls.name) AS id, o.id AS id_site, ls.name AS main_lichenspecies, COUNT(l.id) AS frequency FROM main_observation o JOIN main_tree t ON o.id = t.observation_id JOIN main_lichen l ON o.id = l.observation_id JOIN main_lichenspecies ls ON l.species_id = ls.id GROUP BY o.id, ls.name ORDER BY o.id, frequency DESC;