feat: add filters in jobs status page

src/aind_data_transfer_service/models.py

@@ -1,9 +1,10 @@
 """Module for data models used in application"""
 
-from datetime import datetime
+import ast
+from datetime import datetime, timedelta, timezone
 from typing import List, Optional
 
-from pydantic import AwareDatetime, BaseModel, Field
+from pydantic import AwareDatetime, BaseModel, Field, field_validator
 
 
 class AirflowDagRun(BaseModel):
@@ -37,12 +38,30 @@ class AirflowDagRunsRequestParameters(BaseModel):
 
     limit: int = 25
     offset: int = 0
-    order_by: str = "-start_date"
+    state: Optional[list[str]] = []
+    execution_date_gte: Optional[str] = (
+        datetime.now(timezone.utc) - timedelta(weeks=2)
+    ).isoformat()
+    execution_date_lte: Optional[str] = None
+    order_by: str = "-execution_date"
+
+    @field_validator("execution_date_gte", mode="after")
+    def validate_min_execution_date(cls, execution_date_gte: str):
+        """Validate the earliest submit date filter is within 2 weeks"""
+        min_execution_date = datetime.now(timezone.utc) - timedelta(weeks=2)
+        if datetime.fromisoformat(execution_date_gte) < min_execution_date:
+            raise ValueError(
+                "execution_date_gte must be within the last 2 weeks"
+            )
+        return execution_date_gte
 
     @classmethod
     def from_query_params(cls, query_params: dict):
         """Maps the query parameters to the model"""
-        return cls(**query_params)
+        params = dict(query_params)
+        if "state" in params:
+            params["state"] = ast.literal_eval(params["state"])
+        return cls(**params)
 
 
 class JobStatus(BaseModel):

src/aind_data_transfer_service/server.py

@@ -30,6 +30,7 @@
 from aind_data_transfer_service.hpc.client import HpcClient, HpcClientConfigs
 from aind_data_transfer_service.hpc.models import HpcJobSubmitSettings
 from aind_data_transfer_service.models import (
+    AirflowDagRun,
     AirflowDagRunsRequestParameters,
     AirflowDagRunsResponse,
     JobStatus,
@@ -390,37 +391,58 @@ async def get_job_status_list(request: Request):
     """Get status of jobs with default pagination of limit=25 and offset=0."""
     # TODO: Use httpx async client
     try:
-        params = AirflowDagRunsRequestParameters.from_query_params(
-            request.query_params
-        )
-        params_dict = json.loads(params.model_dump_json())
+        url = os.getenv("AIND_AIRFLOW_SERVICE_JOBS_URL", "").strip("/")
+        get_one_job = request.query_params.get("dag_run_id") is not None
+        if get_one_job:
+            dag_run_id = request.query_params["dag_run_id"]
+        else:
+            params = AirflowDagRunsRequestParameters.from_query_params(
+                request.query_params
+            )
+            params_dict = json.loads(params.model_dump_json())
+        # Send request to Airflow to ListDagRuns or GetDagRun
         response_jobs = requests.get(
-            url=os.getenv("AIND_AIRFLOW_SERVICE_JOBS_URL"),
+            url=f"{url}/{dag_run_id}" if get_one_job else url,
             auth=(
                 os.getenv("AIND_AIRFLOW_SERVICE_USER"),
                 os.getenv("AIND_AIRFLOW_SERVICE_PASSWORD"),
             ),
-            params=params_dict,
+            params=None if get_one_job else params_dict,
         )
         status_code = response_jobs.status_code
         if response_jobs.status_code == 200:
-            dag_runs = AirflowDagRunsResponse.model_validate_json(
-                json.dumps(response_jobs.json())
-            )
+            if get_one_job:
+                dag_run = AirflowDagRun.model_validate_json(
+                    json.dumps(response_jobs.json())
+                )
+                dag_runs = AirflowDagRunsResponse(
+                    dag_runs=[dag_run], total_entries=1
+                )
+            else:
+                dag_runs = AirflowDagRunsResponse.model_validate_json(
+                    json.dumps(response_jobs.json())
+                )
             job_status_list = [
                 JobStatus.from_airflow_dag_run(d) for d in dag_runs.dag_runs
             ]
             message = "Retrieved job status list from airflow"
             data = {
-                "params": params_dict,
+                "params": (
+                    {"dag_run_id": dag_run_id} if get_one_job else params_dict
+                ),
                 "total_entries": dag_runs.total_entries,
                 "job_status_list": [
                     json.loads(j.model_dump_json()) for j in job_status_list
                 ],
             }
         else:
             message = "Error retrieving job status list from airflow"
-            data = {"params": params_dict, "errors": [response_jobs.json()]}
+            data = {
+                "params": (
+                    {"dag_run_id": dag_run_id} if get_one_job else params_dict
+                ),
+                "errors": [response_jobs.json()],
+            }
     except ValidationError as e:
         logging.error(e)
         status_code = 406
@@ -486,13 +508,17 @@ async def jobs(request: Request):
     default_offset = AirflowDagRunsRequestParameters.model_fields[
         "offset"
     ].default
+    default_state = AirflowDagRunsRequestParameters.model_fields[
+        "state"
+    ].default
     return templates.TemplateResponse(
         name="job_status.html",
         context=(
             {
                 "request": request,
                 "default_limit": default_limit,
                 "default_offset": default_offset,
+                "default_state": default_state,
                 "project_names_url": os.getenv(
                     "AIND_METADATA_SERVICE_PROJECT_NAMES_URL"
                 ),

src/aind_data_transfer_service/templates/job_status.html

@@ -3,8 +3,12 @@
 <head>
     <meta charset="UTF-8">
     <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-icons/1.8.1/font/bootstrap-icons.min.css" rel="stylesheet">
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.3/moment.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/latest/jquery.min.js"></script>
+    <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
+    <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css" />
     <title>{% block title %} {% endblock %} AIND Data Transfer Service Jobs</title>
     <style>
         body {
@@ -16,7 +20,7 @@
         }
         .content {
             width: 100%;
-            height: calc(100vh - 200px);
+            height: calc(100vh - 250px);
             iframe {
                 border: none;
                 width: 100%;
@@ -35,7 +39,56 @@
     </nav>
     <div class="content">
         <!-- display total entries from child iframe -->
-        <h2 class="mb-3">Jobs Submitted: <span id="jobs-iframe-total-entries"></h2>
+        <h2 class="mb-2">Jobs Submitted: <span id="jobs-iframe-total-entries"></h2>
+        <!-- filters for job status results-->
+        <div class="card mb-4" style="width:400px">
+            <div class="card-header py-1" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-filters" aria-expanded="false" aria-controls="collapse-filters">
+                <i class="bi bi-filter"></i><span class="ms-2">Filter by</span>
+                <i class="bi bi-chevron-expand float-end"></i>
+            </div>
+            <div id="collapse-filters" class="collapse card-body p-2">
+                <!-- filter by job status -->
+                <div class="input-group input-group-sm mb-1">
+                    <span class="input-group-text" style="width:35%">Status</span>
+                    <select class="form-select" onchange="filterJobsByStatus(this.value);this.blur();">
+                        {% for s in [
+                            {"label": "all", "value": [], "class": "text-dark"},
+                            {"label": "queued", "value": ["queued"], "class": "text-secondary"},
+                            {"label": "running", "value": ["running"], "class": "text-info"},
+                            {"label": "failed", "value": ["failed"], "class": "text-danger"},
+                            {"label": "success", "value": ["success"], "class": "text-success"},
+                        ] %}
+                        {% if s.value==default_state %}<option class="{{ s.class }}" value="{{ s.value }}" selected>{{ s.label }}</option>
+                        {% else %}<option class="{{ s.class }}" value="{{ s.value }}">{{ s.label }}</option>
+                        {% endif %}
+                        {% endfor %}
+                    </select>
+                </div>
+                <!-- filter by job submitted date range -->
+                <div class="input-group input-group-sm">
+                    <span class="input-group-text" style="width:35%">Submit Time</span>
+                    <input id="submit-date-range" class="form-select" type="text" />
+                </div>
+                <!-- search by job id (exact match only) -->
+                <div class="d-flex align-items-center">
+                    <hr class="flex-grow-1 border-secondary">
+                    <span class="mx-2"><small class="d-block">or</small></span>
+                    <hr class="flex-grow-1 border-secondary">
+                </div>
+                <form onsubmit="searchByJobId(event)">
+                    <div class="input-group input-group-sm">
+                        <span class="input-group-text" style="width:35%">Job ID</span>
+                        <input id="job-id-input" type="text" class="form-control" placeholder="exact match only">
+                        <button class="btn btn-outline-secondary" type="submit" title="Search">
+                            <i class="bi bi-search"></i>
+                        </button>
+                        <button class="btn btn-outline-secondary" type="button" onclick="clearJobIdResult(event)" title="Clear">
+                            <i class="bi bi-x-lg"></i>
+                        </button>
+                    </div>
+                </form>
+            </div>
+        </div>
         <!-- toolbar to change jobs per page and navigate pages -->
         <div id="jobs-toolbar" class="btn-toolbar justify-content-between mb-2" role="toolbar">
             <div class="input-group">
@@ -65,7 +118,7 @@ <h2 class="mb-3">Jobs Submitted: <span id="jobs-iframe-total-entries"></h2>
             </div>
         </div>
         <!-- iframe to display paginated job status table -->
-        <iframe id="jobs-iframe" src="{{ url_for('job_status_table').include_query_params(limit=default_limit, offset=default_offset)}}"></iframe>
+        <iframe id="jobs-iframe" src=""></iframe>
     </div>
     <script>
         const PaginateTo = {
@@ -74,12 +127,80 @@ <h2 class="mb-3">Jobs Submitted: <span id="jobs-iframe-total-entries"></h2>
             FIRST: 'first',
             LAST: 'last'
         };
-        function updateJobStatusTableLimit(newLimit) {
+        $(document).ready(function() {
+            const today = moment();
+            const twoWeeksAgo = moment().subtract(13, 'days');
+
+            // initialize daterangepicker for submit date filter
+            $('#submit-date-range').daterangepicker({
+                startDate: twoWeeksAgo,
+                endDate: today,
+                minDate: twoWeeksAgo,
+                maxDate: today,
+                ranges: {
+                    'Today': [today, today],
+                    'Last 3 Days': [moment().subtract(2, 'days'), today],
+                    'Last 7 Days': [moment().subtract(6, 'days'), today],
+                    'Last 14 Days': [twoWeeksAgo, today],
+                }
+            }, filterJobsBySubmitTimeRange);
+
+            // initialize job status table with default values
+            const initialParams = {
+                limit: "{{ default_limit }}",
+                offset: "{{ default_offset }}",
+                state: "{{ default_state }}",
+                // set default submit date range client-side for browser's local time
+                execution_date_gte: twoWeeksAgo.startOf('day').toISOString(),
+                execution_date_lte: today.endOf('day').toISOString(),
+            };
+            updateJobStatusTable(initialParams, false, true);
+        });
+        // EVENT HANDLERS ------------------------------------------------
+        function updateJobStatusTable(newParams, isSearchJobId=false, isInitial=false) {
             var iframe = document.getElementById('jobs-iframe');
-            var currentUrl = new URL(iframe.src);
-            currentUrl.searchParams.set('limit', newLimit);
+            var currentUrl = isInitial ?  new URL("{{ url_for('job_status_table') }}") : new URL(iframe.src);
+            Object.entries(newParams).forEach(([key, value]) => {
+                currentUrl.searchParams.set(key, value);
+            });
+            // reset job_id filter
+            if (!isSearchJobId) {
+                document.getElementById('job-id-input').value = '';
+                currentUrl.searchParams.delete('dag_run_id');
+            }
             iframe.src = currentUrl.toString();
         }
+        // Filters
+        function filterJobsByStatus(newStatus) {
+            updateJobStatusTable({
+                state: newStatus,
+                offset: 0 // reset to first page
+            });
+        }
+        function filterJobsBySubmitTimeRange(start, end) {
+            // NOTE: daterangepicker already has 00:00:00.000 and 23:59:59.999 for start and end
+            updateJobStatusTable({
+                execution_date_gte: start.toISOString(),
+                execution_date_lte: end.toISOString(),
+                offset: 0 // reset to first page
+            });
+        }
+        function searchByJobId(event) {
+            event.preventDefault();
+            // search by job id and reset to first page
+            const jobId = document.getElementById('job-id-input').value;
+            console.log('Searching for job ID:', jobId);
+            updateJobStatusTable({ dag_run_id: jobId, offset: 0}, true);
+        }
+        function clearJobIdResult(event) {
+            event.preventDefault();
+            // clear job id filter and reset to first page
+            updateJobStatusTable({ offset: 0 }, false);
+        }
+        // Pagination
+        function updateJobStatusTableLimit(newLimit) {
+            updateJobStatusTable({limit: newLimit});
+        }
         function updateJobStatusTablePage(paginateTo) {
             var iframe = document.getElementById('jobs-iframe');
             var currentUrl = new URL(iframe.src);
@@ -101,8 +222,7 @@ <h2 class="mb-3">Jobs Submitted: <span id="jobs-iframe-total-entries"></h2>
                     offset = Math.floor(totalEntries / limit) * limit;
                     break;
             }
-            currentUrl.searchParams.set('offset', offset);
-            iframe.src = currentUrl.toString();
+            updateJobStatusTable({offset: offset});
         }
     </script>
     </body>

src/aind_data_transfer_service/templates/job_status_table.html

@@ -27,6 +27,7 @@
       <td class="{% if job_status.job_state == 'success' %}table-success
                 {% elif job_status.job_state == 'failed' %}table-danger
                 {% elif job_status.job_state == 'running' %}table-info
+                {% elif job_status.job_state == 'queued' %}table-secondary
                 {% endif %}">{{job_status.job_state}}</td>
       <td class="datetime_to_be_adjusted">{{job_status.submit_time}}</td>
       <td class="datetime_to_be_adjusted">{{job_status.start_time}}</td>
@@ -63,10 +64,12 @@ <h4 class="alert-heading">{{ message }}</h4>
         limit = parseInt('{{limit}}');
         total_entries = parseInt('{{total_entries}}');
         parent.document.getElementById('jobs-iframe-total-entries').innerText = '{{total_entries}}';
-        parent.document.getElementById('jobs-iframe-showing').innerText = `${offset + 1} to ${Math.min(offset + limit, total_entries)} of ${total_entries}`;
+        parent.document.getElementById('jobs-iframe-showing').innerText = (total_entries > 1)
+          ? `${offset + 1} to ${Math.min(offset + limit, total_entries)} of ${total_entries}`
+          : `${total_entries} to ${total_entries} of ${total_entries}`;
         // also update the pagination buttons
-        isFirst = (offset == 0);
-        isLast = (offset + limit >= total_entries);
+        isFirst = (total_entries > 1) ? (offset == 0) : true;
+        isLast = (total_entries > 1) ? (offset + limit >= total_entries) : true;
         parent.document.getElementById('jobs-page-btn-first').disabled = isFirst;
         parent.document.getElementById('jobs-page-btn-prev').disabled = isFirst;
         parent.document.getElementById('jobs-page-btn-next').disabled = isLast;