Merge branch 'main' into merge_in_df_docDB

.github/workflows/publish.yaml

@@ -42,10 +42,10 @@ jobs:
       - uses: actions/checkout@master
       - name: Pull latest changes
         run: git pull origin main
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Configure aws credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:

.streamlit/config.toml

@@ -1,2 +1,8 @@
 [theme]
-base="dark"
+base="dark"
+
+[server]
+fileWatcherType = "poll"
+
+[global]
+disableWidgetStateDuplicationWarning = true

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim
+FROM python:3.9-slim
 
 WORKDIR /app
 

code/Home.py

@@ -12,50 +12,44 @@
 
 """
 
-__ver__ = 'v2.4.1'
+import os
 
+import extra_streamlit_components as stx
+import numpy as np
 import pandas as pd
 import streamlit as st
-import numpy as np
-import os
-
 import streamlit_nested_layout
-from streamlit_plotly_events import plotly_events
+from aind_auto_train import __version__ as auto_train_version
+from aind_auto_train.auto_train_manager import DynamicForagingAutoTrainManager
+from aind_auto_train.curriculum_manager import CurriculumManager
 from pygwalker.api.streamlit import StreamlitRenderer, init_streamlit_comm
-import extra_streamlit_components as stx
-
-from util.settings import draw_type_mapper_session_level, draw_type_layout_definition
-
-from util.streamlit import (aggrid_interactive_table_session,
-                            aggrid_interactive_table_curriculum, add_session_filter, data_selector,
-                            add_xy_selector, add_xy_setting, add_auto_train_manager, add_dot_property_mapper,
-                            _plot_population_x_y)
-from util.aws_s3 import (
-    load_data,
-    draw_session_plots_quick_preview,
-    show_session_level_img_by_key_and_prefix,
-    show_debug_info,
-)
-from util.url_query_helper import (
-    sync_URL_to_session_state, sync_session_state_to_URL,
-    slider_wrapper_for_url_query, checkbox_wrapper_for_url_query,
-    multiselect_wrapper_for_url_query, number_input_wrapper_for_url_query,
-)
-
+from streamlit_plotly_events import plotly_events
+from util.aws_s3 import (draw_session_plots_quick_preview, load_data,
+                         show_debug_info,
+                         show_session_level_img_by_key_and_prefix)
 from util.fetch_data_docDB import load_data_from_docDB
-
-from aind_auto_train.curriculum_manager import CurriculumManager
-from aind_auto_train.auto_train_manager import DynamicForagingAutoTrainManager
-from aind_auto_train import __version__ as auto_train_version
-
+from util.settings import (draw_type_layout_definition,
+                           draw_type_mapper_session_level)
+from util.streamlit import (_plot_population_x_y, add_auto_train_manager,
+                            add_dot_property_mapper, add_session_filter,
+                            add_xy_selector, add_xy_setting,
+                            aggrid_interactive_table_curriculum,
+                            aggrid_interactive_table_session, data_selector,
+                            add_footnote)
+from util.url_query_helper import (checkbox_wrapper_for_url_query,
+                                   multiselect_wrapper_for_url_query,
+                                   number_input_wrapper_for_url_query,
+                                   slider_wrapper_for_url_query,
+                                   sync_session_state_to_URL,
+                                   sync_URL_to_session_state)
 
 try:
     st.set_page_config(layout="wide", 
                     page_title='Foraging behavior browser',
                     page_icon=':mouse2:',
                         menu_items={
-                        'Report a bug': "https://github.com/hanhou/foraging-behavior-browser/issues",
-                        'About': "Github repo: https://github.com/hanhou/foraging-behavior-browser/"
+                        'Report a bug': "https://github.com/AllenNeuralDynamics/foraging-behavior-browser/issues",
+                        'About': "Github repo: https://github.com/AllenNeuralDynamics/foraging-behavior-browser"
                         }
                     )
 except:
@@ -456,6 +450,20 @@ def _get_data_source(rig):
 
     # --- Load data from docDB ---
     _df = merge_in_df_docDB(_df)
+
+    # add docDB_status column
+    _df["docDB_status"] = _df.apply(
+        lambda row: (
+            "0_not uploaded"
+            if pd.isnull(row["location"])
+            else (
+                "1_uploaded but not processed"
+                if pd.isnull(row["results_location"])
+                else "2_uploaded and processed"
+            )
+        ),
+        axis=1,
+    )
 
     st.session_state.df['sessions_bonsai'] = _df  # Somehow _df loses the reference to the original dataframe
     st.session_state.session_stats_names = [keys for keys in _df.keys()]
@@ -498,19 +506,14 @@ def app():
         add_session_filter(if_bonsai=True,
                            url_query=url_query)
         data_selector()
-
-        st.markdown('---')
-        st.markdown(f'#### Han Hou @ 2024 {__ver__}')
-        st.markdown('[bug report / feature request](https://github.com/AllenNeuralDynamics/foraging-behavior-browser/issues)')
+        add_footnote()
 
         with st.expander('Debug', expanded=False):
-            if st.button('Reload data from AWS S3'):
+            if st.button('Clear session state and reload data'):
                 st.cache_data.clear()
                 init()
                 st.rerun()
 
-
-
     with st.container():
         # col1, col2 = st.columns([1.5, 1], gap='small')
         # with col1:

code/__init__.py

@@ -0,0 +1 @@
+__ver__ = 'v2.5.4'

code/pages/1_Playground.py → code/pages/1_Learning trajectory.py

@@ -1,15 +1,14 @@
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
 import s3fs
 import streamlit as st
+from plotly.subplots import make_subplots
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
-import pandas as pd
-import numpy as np
-import plotly.graph_objects as go
-from plotly.subplots import make_subplots
 from streamlit_plotly_events import plotly_events
-
-from util.streamlit import add_session_filter, data_selector
 from util.aws_s3 import load_data
+from util.streamlit import add_session_filter, data_selector
 
 ss = st.session_state
 

code/pages/2_HMM-GLM.py

@@ -4,12 +4,14 @@
 """
 import os
 import re
-import numpy as np
-from PIL import Image
 
-import streamlit as st
+import numpy as np
 import s3fs
+import streamlit as st
 import streamlit_nested_layout
+from PIL import Image
+
+from util.streamlit import add_footnote
 
 try:
     st.set_page_config(layout="wide", 
@@ -65,7 +67,9 @@ def app():
         if st.button('Reload data from S3'):
             st.cache_data.clear()
             st.rerun()
-
+
+        add_footnote()
+
     data_folder_selected = widget_data_folder.selectbox('Select Data Folder', data_folders)
 
     if data_folder_selected:

code/pages/3_AIND data access playground.py

@@ -2,9 +2,9 @@
 '''
 
 import logging
+
 import streamlit as st
 from streamlit_dynamic_filters import DynamicFilters
-
 from util.fetch_data_docDB import load_data_from_docDB
 
 try:
@@ -21,6 +21,8 @@
 
 df = load_data_from_docDB()
 
+st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
+
 dynamic_filters = DynamicFilters(
     df=df, 
     filters=['subject_id', 'subject_genotype'])