Engaging Crowds: Subject set completeness

packages/app-project/package.json

@@ -51,6 +51,7 @@
     "morgan": "^1.10.0",
     "newrelic": "~7.1.0",
     "next": "~9.5.5",
+    "node-fetch": "~2.6.1",
     "panoptes-client": "~3.2.1",
     "path-match": "~1.2.4",
     "polished": "~3.6.4",

packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js

@@ -0,0 +1,65 @@
+import { panoptes } from '@zooniverse/panoptes-js'
+import fetch from 'node-fetch'
+
+import { logToSentry } from '@helpers/logger'
+
+async function fetchSubjectSetData(subjectSetIDs, env) {
+  let subject_sets = []
+  try {
+    const query = {
+      env,
+      id: subjectSetIDs.join(',')
+    }
+    const response = await panoptes.get('/subject_sets', query)
+    subject_sets = response.body.subject_sets
+    await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env)))
+  } catch (error) {
+    console.error(error)
+    logToSentry(error)
+  }
+  return subject_sets
+}
+
+async function fetchWorkflowCellectStatus(workflow) {
+  let groups = {}
+  if (workflow.grouped) {
+    try {
+      const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status`
+      const response = await fetch(workflowURL)
+      const body = await response.json()
+      groups = body.groups ?? {}
+    } catch (error) {
+      console.error(error)
+      logToSentry(error)
+    }
+  }
+  return groups
+}
+
+async function fetchPreviewImage (subjectSet, env) {
+  try {
+    const response = await panoptes
+      .get('/set_member_subjects', {
+        env,
+        subject_set_id: subjectSet.id,
+        include: 'subject',
+        page_size: 1
+      })
+    const { linked } = response.body
+    subjectSet.subjects = linked.subjects
+  } catch (error) {
+    console.error(error)
+    logToSentry(error)
+  }
+}
+
+export default async function workflowSubjectSets(workflow, env) {
+  const subjectSetCounts = await fetchWorkflowCellectStatus(workflow)
+  const subjectSetIDs = Object.keys(subjectSetCounts)
+  const subjectSets = await fetchSubjectSetData(subjectSetIDs, env)
+  subjectSets.forEach(subjectSet => {
+    subjectSet.availableSubjects = subjectSetCounts[subjectSet.id]
+  })
+  return subjectSets
+}
+

packages/app-project/src/helpers/fetchSubjectSets/index.js

@@ -0,0 +1 @@
+export { default } from './fetchSubjectSets'

packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js

@@ -1,67 +1,72 @@
 import { panoptes } from '@zooniverse/panoptes-js'
+import fetch from 'node-fetch'
+
+import { logToSentry } from '@helpers/logger'
+import fetchSubjectSets from '@helpers/fetchSubjectSets'
 
 async function fetchWorkflowData (activeWorkflows, env) {
-  const response = await panoptes
-    .get('/workflows', {
+  try {
+    const query = {
       complete: false,
       env,
-      fields: 'completeness,grouped',
-      id: activeWorkflows.join(','),
-      include: 'subject_sets'
-    })
-  const { workflows, linked } = response.body
-  const subjectSets = linked ? linked.subject_sets : []
-  await Promise.allSettled(subjectSets.map(subjectSet => fetchPreviewImage(subjectSet, env)))
-  return { subjectSets, workflows }
+      fields: 'completeness,display_name,grouped',
+      id: activeWorkflows.join(',')
+    }
+    const response = await panoptes.get('/workflows', query)
+    return response.body.workflows
+  } catch (error) {
+    logToSentry(error)
+    throw error
+  }
 }
-
-function fetchDisplayNames (language, activeWorkflows, env) {
-  return panoptes
-    .get('/translations', {
+async function fetchDisplayNames (language, activeWorkflows, env) {
+  let displayNames = {}
+  try {
+    const response = await panoptes.get('/translations', {
       env,
       fields: 'strings,translated_id',
       language,
       'translated_id': activeWorkflows.join(','),
       'translated_type': 'workflow'
     })
-    .then(response => response.body.translations)
-    .then(createDisplayNamesMap)
+    const { translations } = response.body
+    displayNames = createDisplayNamesMap(translations)
+  } catch (error) {
+    logToSentry(error)
+    throw error
+  }
+  return displayNames
 }
 
-async function fetchPreviewImage (subjectSet, env) {
-  const response = await panoptes
-    .get('/set_member_subjects', {
-      env,
-      subject_set_id: subjectSet.id,
-      include: 'subject',
-      page_size: 1
-    })
-  const { linked } = response.body
-  subjectSet.subjects = linked.subjects
+async function buildWorkflow(workflow, displayName, isDefault, env) {
+  const workflowData = {
+    completeness: workflow.completeness || 0,
+    default: isDefault,
+    displayName,
+    grouped: workflow.grouped,
+    id: workflow.id,
+    subjectSets: []
+  }
+  if (workflow.grouped) {
+    workflowData.subjectSets = await fetchSubjectSets(workflow, env)
+  }
+
+  return workflowData
 }
 
 async function fetchWorkflowsHelper (language = 'en', activeWorkflows, defaultWorkflow, env) {
-  const { subjectSets, workflows } = await fetchWorkflowData(activeWorkflows, env)
+  const workflows = await fetchWorkflowData(activeWorkflows, env)
   const workflowIds = workflows.map(workflow => workflow.id)
   const displayNames = await fetchDisplayNames(language, workflowIds, env)
 
-  return workflows.map(workflow => {
+  const awaitWorkflows = workflows.map(workflow => {
     const isDefault = workflows.length === 1 || workflow.id === defaultWorkflow
-    const workflowSubjectSets = workflow.links.subject_sets
-      .map(subjectSetID => {
-        return subjectSets.find(subjectSet => subjectSet.id === subjectSetID)
-      })
-      .filter(Boolean)
-
-    return {
-      completeness: workflow.completeness || 0,
-      default: isDefault,
-      displayName: displayNames[workflow.id],
-      grouped: workflow.grouped,
-      id: workflow.id,
-      subjectSets: workflowSubjectSets
-    }
+    const displayName = displayNames[workflow.id] || workflow.display_name
+    return buildWorkflow(workflow, displayName, isDefault, env)
   })
+  const workflowStatuses = await Promise.allSettled(awaitWorkflows)
+  const workflowsWithSubjectSets = workflowStatuses.map(result => result.value || result.reason)
+  return workflowsWithSubjectSets
 }
 
 function createDisplayNamesMap (translations) {