diff --git a/src/aind_qc_portal/docdb/database.py b/src/aind_qc_portal/docdb/database.py
index 5f2f89f..6c0239b 100644
--- a/src/aind_qc_portal/docdb/database.py
+++ b/src/aind_qc_portal/docdb/database.py
@@ -129,8 +129,8 @@ def get_all():
@pn.cache(ttl=TIMEOUT_1H)
-def get_project(project: str):
- filter = {"data_description.project_name": project}
+def get_project(project_name: str):
+ filter = {"data_description.project_name": project_name}
limit = 0
paginate_batch_size = 500
response = client.retrieve_docdb_records(
@@ -145,6 +145,9 @@ def get_project(project: str):
"session.session_start_time": 1,
"data_description.data_level": 1,
"data_description.project_name": 1,
+ "rig.rig_id": 1,
+ "session.experimenter_full_name": 1,
+ "quality_control": 1,
},
limit=limit,
paginate_batch_size=paginate_batch_size,
@@ -153,6 +156,36 @@ def get_project(project: str):
return response
+@pn.cache(ttl=TIMEOUT_1H)
+def get_project_custom(project_name: str, fields: list):
+ """Get all records that match a project name, with custom fields
+
+ Parameters
+ ----------
+ project_name : str
+ fields : list
+ List of fields to retain from DocDB record
+
+ Returns
+ -------
+ list
+ List of dictionaries containing the fields requested
+ """
+ filter = {"data_description.project_name": project_name}
+ limit = 0
+ paginate_batch_size = 500
+ response = client.retrieve_docdb_records(
+ filter_query=filter,
+ projection={
+ "_id": 1,
+ } | {field: 1 for field in fields},
+ limit=limit,
+ paginate_batch_size=paginate_batch_size,
+ )
+
+ return response
+
+
@pn.cache
def get_subjects():
filter = {
diff --git a/src/aind_qc_portal/panel/metric.py b/src/aind_qc_portal/panel/metric.py
index 1705244..825d60c 100644
--- a/src/aind_qc_portal/panel/metric.py
+++ b/src/aind_qc_portal/panel/metric.py
@@ -168,7 +168,7 @@ def metric_panel(self):
options=["Pass", "Fail", "Pending"],
name="Metric status",
)
-
+
if pn.state.user == "guest":
self.state_selector.disabled = True
else:
diff --git a/src/aind_qc_portal/panel/quality_control.py b/src/aind_qc_portal/panel/quality_control.py
index 495220e..f3a3dbe 100644
--- a/src/aind_qc_portal/panel/quality_control.py
+++ b/src/aind_qc_portal/panel/quality_control.py
@@ -213,8 +213,6 @@ def status_panel(self):
def panel(self):
"""Build a Panel object representing this QC action"""
- if not self._has_data or not self._data:
- return pn.widgets.StaticText(value="No QC object available")
# build the header
md = f"""
@@ -222,6 +220,13 @@ def panel(self):
"""
header = pn.pane.Markdown(md)
+ if not self._has_data or not self._data:
+ return pn.Row(
+ pn.HSpacer(),
+ pn.Column(header,
+ pn.widgets.StaticText(value="No QC object available", styles={"font-size": "22pt"}), styles=OUTER_STYLE),
+ pn.HSpacer())
+
# build the display box: this shows the current state in DocDB of this asset
# if any evaluations are failing, we'll show a warning
failing_eval_str = ""
diff --git a/src/aind_qc_portal/projects/dataset.py b/src/aind_qc_portal/projects/dataset.py
new file mode 100644
index 0000000..67e9775
--- /dev/null
+++ b/src/aind_qc_portal/projects/dataset.py
@@ -0,0 +1,114 @@
+import pandas as pd
+import param
+
+from aind_qc_portal.docdb.database import get_project, get_project_custom
+from aind_qc_portal.utils import format_link, qc_color
+from aind_data_schema.core.quality_control import QualityControl, Status
+
+class ProjectDataset(param.Parameterized):
+ """Generic dataset class, loads default data for all projects"""
+ subject_filter = param.String(default="")
+
+ def __init__(self, project_name: str):
+ """Create a ProjectDataset object"""
+
+ self.project_name = project_name
+ self._df = None
+ self.exposed_columns = [
+ "subject_id", "Date", "name", "Operator", "S3 link", "Status", "Subject view", "QC view", "session_type", "raw"
+ ]
+ self._get_assets()
+
+ def _get_assets(self):
+ """Get all assets with this project name"""
+ print(self.project_name)
+ records = get_project(self.project_name)
+
+ data = []
+ for record in records:
+ subject_id = record.get('subject', {}).get('subject_id')
+
+ # rig, operator, QC notes get bubbled up? qc status,
+ # custom genotype mapping. Do this for learning-mfish
+
+ # reconstruct the QC object, if possible
+ if record.get('quality_control'):
+ qc = QualityControl(**record.get('quality_control'))
+ else:
+ qc = None
+
+ record_data = {
+ '_id': record.get('_id'),
+ 'raw': record.get('data_description', {}).get('data_level') == 'raw',
+ 'project_name': record.get('data_description', {}).get('project_name'),
+ 'location': record.get('location'),
+ 'name': record.get('name'),
+ 'session_start_time': record.get('session', {}).get('session_start_time'),
+ 'session_type': record.get('session', {}).get('session_type'),
+ 'subject_id': subject_id,
+ 'operator': list(record.get('session', {}).get('experimenter_full_name')),
+ 'Status': qc.status().value if qc else "No QC",
+ }
+ data.append(record_data)
+
+ if len(data) == 0:
+ self._df = None
+ return
+
+ self._df = pd.DataFrame(data)
+ self._df["timestamp"] = pd.to_datetime(self._df["session_start_time"], format='mixed', utc=True)
+ self._df["Date"] = self._df["timestamp"].dt.strftime("%Y-%m-%d %H:%M:%S")
+ self._df["S3 link"] = self._df["location"].apply(lambda x: format_link(x, text="S3 link"))
+ self._df["Subject view"] = self._df["_id"].apply(lambda x: format_link(f"/qc_asset_app?id={x}"))
+ self._df["qc_link"] = self._df["_id"].apply(lambda x: f"/qc_app?id={x}")
+ self._df["QC view"] = self._df.apply(lambda row: format_link(row["qc_link"]), axis=1)
+ self._df["Operator"] = self._df["operator"].apply(lambda x: ", ".join(x))
+ self._df.sort_values(by="timestamp", ascending=True, inplace=True)
+ self._df.sort_values(by="subject_id", ascending=True, inplace=True)
+
+ def filtered_data(self):
+ if self.subject_filter:
+ filtered_df = self._df[self._df["subject_id"].str.contains(self.subject_filter, case=False, na=False)]
+ else:
+ filtered_df = self._df
+
+ return filtered_df[self.exposed_columns]
+
+ @property
+ def data(self):
+
+ return self.filtered_data()[self.exposed_columns].style.map(qc_color, subset=["Status"])
+
+
+ @property
+ def timestamp_data(self):
+ if self.subject_filter:
+ filtered_df = self._df[self._df["subject_id"].str.contains(self.subject_filter, case=False, na=False)]
+ else:
+ filtered_df = self._df
+
+ return filtered_df[["timestamp"]]
+
+
+class LearningmFishDataset(ProjectDataset):
+
+ def __init__(self, project_name: str):
+ if project_name != "Learning mFISH-V1omFISH":
+ raise ValueError("This class is only for Learning mFISH-V1omFISH")
+
+ super().__init__(project_name=project_name)
+
+ self._get_mfish_assets()
+
+ def _get_mfish_assets(self):
+ """Load additional information needed for the Learning mFISH-V1omFISH project
+
+ Extra data should be appended to the self._df dataframe and then needs to be added to the
+ list of exposed columns.
+ """
+ data = get_project_custom(self.project_name, [""])
+
+
+mapping = {
+ "Learning mFISH-V1omFISH": LearningmFishDataset,
+}
diff --git a/src/aind_qc_portal/projects/project_view.py b/src/aind_qc_portal/projects/project_view.py
new file mode 100644
index 0000000..187f5b2
--- /dev/null
+++ b/src/aind_qc_portal/projects/project_view.py
@@ -0,0 +1,103 @@
+from typing import Optional
+import pandas as pd
+import panel as pn
+import altair as alt
+
+from aind_qc_portal.projects.dataset import mapping, ProjectDataset
+from aind_qc_portal.utils import df_timestamp_range, OUTER_STYLE
+from aind_qc_portal.utils import OUTER_STYLE, AIND_COLORS
+
+
+class ProjectView():
+
+ def __init__(self, project_name: str):
+ cls = mapping.get(project_name, ProjectDataset)
+ self.dataset = cls(project_name=project_name)
+ self.project_name = project_name
+
+ @property
+ def has_data(self):
+ return self.dataset.data is not None
+
+ def get_subjects(self):
+ if not self.has_data:
+ return []
+
+ return self.dataset._df["subject_id"].unique()
+
+ def get_data(self) -> Optional[pd.DataFrame]:
+ if not self.has_data:
+ return None
+
+ return self.dataset.filtered_data()
+
+ def get_data_styled(self):
+ if not self.has_data:
+ return None
+
+ return self.dataset.data
+
+ def history_panel(self):
+ """Create a plot showing the history of this asset, showing how assets were derived from each other"""
+ if not self.has_data:
+ return pn.widgets.StaticText(
+ value=f"No data found for project: {self.project_name}"
+ )
+
+ # Calculate the time range to show on the x axis
+ (min_range, max_range, range_unit, format) = df_timestamp_range(
+ self.dataset.timestamp_data
+ )
+
+ chart = (
+ alt.Chart(self.get_data())
+ .mark_bar()
+ .encode(
+ x=alt.X(
+ "Date:T",
+ title="Time",
+ scale=alt.Scale(domain=[min_range, max_range]),
+ axis=alt.Axis(format=format, tickCount=range_unit),
+ ),
+ y=alt.Y("subject_id:N", title="Subject ID"),
+ tooltip=[
+ "subject_id",
+ "session_type",
+ "Date",
+ ],
+ color=alt.Color("subject_id:N"),
+ href="qc_link:N",
+ )
+ .properties(width=900)
+ )
+
+ return pn.pane.Vega(chart, sizing_mode="stretch_width", styles=OUTER_STYLE)
+
+ def panel(self) -> pn.Column:
+ """Return panel object"""
+
+ md = f"""
+
+ {self.project_name}
+
+ {len(self.dataset.filtered_data())} data assets are associated with this project.
+ """
+
+ header = pn.pane.Markdown(md, width=1000, styles=OUTER_STYLE)
+
+ chart_pane = self.history_panel()
+
+ df_pane = pn.pane.DataFrame(self.get_data_styled(), width=950, escape=False, index=False)
+
+ def update_subject_filter(event):
+ self.dataset.subject_filter = event.new
+ df_pane.object = self.get_data()
+
+ subject_filter = pn.widgets.Select(name="Subject filter", options=list(self.get_subjects()))
+ subject_filter.param.watch(update_subject_filter, "value")
+
+ df_col = pn.Column(subject_filter, df_pane, styles=OUTER_STYLE)
+
+ col = pn.Column(header, chart_pane, df_col)
+
+ return col
diff --git a/src/aind_qc_portal/qc_project_app.py b/src/aind_qc_portal/qc_project_app.py
index 1dcc79c..18133e5 100644
--- a/src/aind_qc_portal/qc_project_app.py
+++ b/src/aind_qc_portal/qc_project_app.py
@@ -5,11 +5,11 @@
Intended to be the main entry point for a project's data
"""
import panel as pn
-import altair as alt
import param
-import pandas as pd
-from aind_qc_portal.docdb.database import get_project, _raw_name_from_derived
-from aind_qc_portal.utils import OUTER_STYLE, set_background, format_link, AIND_COLORS, df_timestamp_range
+from aind_qc_portal.utils import OUTER_STYLE, set_background, AIND_COLORS
+from aind_qc_portal.projects.project_view import ProjectView
+
+pn.extension("vega")
set_background()
@@ -21,132 +21,8 @@ class Settings(param.Parameterized):
settings = Settings()
pn.state.location.sync(settings, {"project_name": "project_name"})
+project_view = ProjectView(settings.project_name)
-class ProjectView(param.Parameterized):
- subject_filter = param.String(default="")
-
- def __init__(self):
- self._get_assets()
-
- def _get_assets(self):
- """Get all assets with this project name"""
- print(settings.project_name)
- records = get_project(settings.project_name)
-
- data = []
- for record in records:
- raw_name = _raw_name_from_derived(record['name'])
- subject_id = record.get('subject', {}).get('subject_id')
-
- record_data = {
- '_id': record.get('_id'),
- 'raw': record.get('data_description', {}).get('data_level') == 'raw',
- 'project_name': record.get('data_description', {}).get('project_name'),
- 'location': record.get('location'),
- 'name': record.get('name'),
- 'session_start_time': record.get('session', {}).get('session_start_time'),
- 'session_type': record.get('session', {}).get('session_type'),
- 'subject_id': subject_id,
- 'genotype': record.get('subject', {}).get('genotype'),
- }
- data.append(record_data)
-
- if len(data) == 0:
- self.data = None
- return
-
- self.data = pd.DataFrame(data)
- self.data["timestamp"] = pd.to_datetime(self.data["session_start_time"], format='mixed', utc=True)
- self.data["Date"] = self.data["timestamp"].dt.strftime("%Y-%m-%d %H:%M:%S")
- self.data["S3 link"] = self.data["location"].apply(lambda x: format_link(x, text="S3 link"))
- self.data["Subject view"] = self.data["_id"].apply(lambda x: format_link(f"/qc_asset_app?id={x}"))
- self.data["QC view"] = self.data["_id"].apply(lambda x: format_link(f"/qc_app?id={x}"))
-
- self.data.sort_values(by="timestamp", ascending=True, inplace=True)
- self.data.sort_values(by="subject_id", ascending=True, inplace=True)
-
- def get_subjects(self):
- if self.data is None:
- return []
-
- return self.data["subject_id"].unique()
-
- def get_data(self):
- if self.data is None:
- return None
-
- if self.subject_filter:
- filtered_df = self.data[self.data["subject_id"].str.contains(self.subject_filter, case=False, na=False)]
- else:
- filtered_df = self.data
-
- return filtered_df[["subject_id", "Date", "S3 link", "Subject view", "QC view", "genotype", "session_type", "raw"]]
-
- def history_panel(self):
- """Create a plot showing the history of this asset, showing how assets were derived from each other"""
- if self.data is None:
- return pn.widgets.StaticText(
- value=f"No data found for project: {settings.project_name}"
- )
-
- # Calculate the time range to show on the x axis
- (min_range, max_range, range_unit, format) = df_timestamp_range(
- self.data
- )
-
- chart = (
- alt.Chart(self.data)
- .mark_bar()
- .encode(
- x=alt.X(
- "Date:T",
- title="Time",
- scale=alt.Scale(domain=[min_range, max_range]),
- axis=alt.Axis(format=format, tickCount=range_unit),
- ),
- y=alt.Y("subject_id:N", title="Subject ID"),
- tooltip=[
- "name",
- "session_type",
- "subject_id",
- "genotype",
- "Date",
- ],
- color=alt.Color("subject_id:N"),
- )
- .properties(width=900)
- )
-
- return pn.pane.Vega(chart, sizing_mode="stretch_width", styles=OUTER_STYLE)
-
-project_view = ProjectView()
-
-md = f"""
-
- {settings.project_name}
-
-{len(project_view.data)} data assets are associated with this project.
-"""
-
-header = pn.pane.Markdown(md, width=1000, styles=OUTER_STYLE)
-
-
-chart_pane = project_view.history_panel()
-
-df_pane = pn.pane.DataFrame(project_view.get_data(), width=950, escape=False, index=False)
-
-
-def update_subject_filter(event):
- project_view.subject_filter = event.new
- df_pane.object = project_view.get_data()
-
-
-subject_filter = pn.widgets.Select(name="Subject filter", options=list(project_view.get_subjects()))
-subject_filter.param.watch(update_subject_filter, "value")
-
-df_col = pn.Column(subject_filter, df_pane, styles=OUTER_STYLE)
-
-col = pn.Column(header, chart_pane, df_col)
-row = pn.Row(pn.HSpacer(), col, pn.HSpacer())
+row = pn.Row(pn.HSpacer(), project_view.panel(), pn.HSpacer())
row.servable(title="AIND QC - Project")
diff --git a/src/aind_qc_portal/utils.py b/src/aind_qc_portal/utils.py
index 65a0c8f..54f7c02 100644
--- a/src/aind_qc_portal/utils.py
+++ b/src/aind_qc_portal/utils.py
@@ -59,8 +59,7 @@ def set_background():
pn.config.raw_css.append(BACKGROUND_CSS)
-def status_html(status: Status):
- print(status)
+def status_color(status: Status):
if status == Status.PASS:
color = AIND_COLORS["green"]
elif status == Status.PENDING:
@@ -69,8 +68,11 @@ def status_html(status: Status):
color = AIND_COLORS["red"]
else:
color = "#756575"
+ return color
- return f'{status.value}'
+
+def status_html(status: Status, text: str = ""):
+ return f'{text if text else status.value}'
def df_timestamp_range(df, column="timestamp"):