From 4f0f54be9496b0c180d4cf2ad616ac0ec1dbbe07 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Wed, 3 Feb 2021 15:03:01 +0000 Subject: [PATCH 01/10] Install node-fetch --- packages/app-project/package.json | 1 + yarn.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/app-project/package.json b/packages/app-project/package.json index 01b2d5a6af..be24565ab4 100644 --- a/packages/app-project/package.json +++ b/packages/app-project/package.json @@ -51,6 +51,7 @@ "morgan": "^1.10.0", "newrelic": "~7.1.0", "next": "~9.5.5", + "node-fetch": "~2.6.1", "panoptes-client": "~3.2.1", "path-match": "~1.2.4", "polished": "~3.6.4", diff --git a/yarn.lock b/yarn.lock index 584b98e9e1..02ab6eef80 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13400,7 +13400,7 @@ node-fetch@2.6.0: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== -node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.5.0, node-fetch@^2.6.0, node-fetch@^2.6.1: +node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.5.0, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@~2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== From 37604f6f05322089ee28028ac24665170547966a Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Wed, 3 Feb 2021 15:04:29 +0000 Subject: [PATCH 02/10] Fetch subject set status from cellect Fetch subject set statuses from Cellect, after fetching workflows from Panoptes. Add `subjectSet.availableSubjects` to each workflow subject set. --- .../fetchWorkflowsHelper.js | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js index 362f04c1ca..34457cca42 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js @@ -1,4 +1,5 @@ import { panoptes } from '@zooniverse/panoptes-js' +import fetch from 'node-fetch' async function fetchWorkflowData (activeWorkflows, env) { const response = await panoptes @@ -28,6 +29,14 @@ function fetchDisplayNames (language, activeWorkflows, env) { .then(createDisplayNamesMap) } +async function fetchWorkflowCellectStatus(workflow) { + const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status` + const response = await fetch(workflowURL) + const body = await response.json() + const { groups } = body ?? {} + return groups +} + async function fetchPreviewImage (subjectSet, env) { const response = await panoptes .get('/set_member_subjects', { @@ -40,28 +49,34 @@ async function fetchPreviewImage (subjectSet, env) { subjectSet.subjects = linked.subjects } +async function buildWorkflow(workflow, displayNames, subjectSets) { + const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) + const workflowSubjectSets = workflow.links.subject_sets + .map(subjectSetID => { + const subjectSet = subjectSets.find(subjectSet => subjectSet.id === subjectSetID) + subjectSet.availableSubjects = subjectSetCounts[subjectSetID] + return subjectSet + }) + .filter(Boolean) + + return { + completeness: workflow.completeness || 0, + displayName: displayNames[workflow.id], + grouped: workflow.grouped, + id: workflow.id, + subjectSets: workflowSubjectSets + } +} + async function fetchWorkflowsHelper (language = 'en', activeWorkflows, defaultWorkflow, env) { const { subjectSets, workflows } = await fetchWorkflowData(activeWorkflows, env) const workflowIds = workflows.map(workflow => workflow.id) const displayNames = await fetchDisplayNames(language, workflowIds, env) - return workflows.map(workflow => { - const isDefault = workflows.length === 1 || workflow.id === defaultWorkflow - const workflowSubjectSets = workflow.links.subject_sets - .map(subjectSetID => { - return subjectSets.find(subjectSet => subjectSet.id === subjectSetID) - }) - .filter(Boolean) - - return { - completeness: workflow.completeness || 0, - default: isDefault, - displayName: displayNames[workflow.id], - grouped: workflow.grouped, - id: workflow.id, - subjectSets: workflowSubjectSets - } - }) + const awaitWorkflows = await Promise.allSettled(workflows.map(workflow => buildWorkflow(workflow, displayNames, subjectSets))) + const workflowsWithSubjectSets = awaitWorkflows.map(result => result.value) + console.log({ workflowsWithSubjectSets }) + return workflowsWithSubjectSets } function createDisplayNamesMap (translations) { From 510d85e856cd7e375e7a33a321a3daf833364da4 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Wed, 3 Feb 2021 15:08:00 +0000 Subject: [PATCH 03/10] Add completeness counts to subject set cards --- .../SubjectSetCard/SubjectSetCard.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/app-project/src/shared/components/SubjectSetPicker/components/SubjectSetCard/SubjectSetCard.js b/packages/app-project/src/shared/components/SubjectSetPicker/components/SubjectSetCard/SubjectSetCard.js index e9b58db79d..22a9c59d69 100644 --- a/packages/app-project/src/shared/components/SubjectSetPicker/components/SubjectSetCard/SubjectSetCard.js +++ b/packages/app-project/src/shared/components/SubjectSetPicker/components/SubjectSetCard/SubjectSetCard.js @@ -6,13 +6,15 @@ import { array, number, string } from 'prop-types' import React from 'react' function SubjectSetCard (props) { - const { display_name, id, set_member_subjects_count, subjects } = props + const { availableSubjects, display_name, id, set_member_subjects_count, subjects } = props const [subject] = subjects const { publicRuntimeConfig = {} } = getConfig() || {} const assetPrefix = publicRuntimeConfig.assetPrefix || '' const placeholderUrl = `${assetPrefix}/subject-placeholder.png` const subjectURLs = subject ? subject.locations.map(location => Object.values(location)[0]) : [] const alt = subject ? `Subject ${subject.id}` : 'Loading' + const completeness = 1 - (availableSubjects / set_member_subjects_count) + const percentComplete = parseInt(100 * completeness) return ( {display_name} - + {`${set_member_subjects_count} subjects`} +
+ + {`${percentComplete}% complete`} + From e0ba688c38a915f53752992259dec84c5415a2dc Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Mon, 8 Feb 2021 16:39:21 +0000 Subject: [PATCH 04/10] Restore workflow.default --- .../fetchWorkflowsHelper/fetchWorkflowsHelper.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js index 34457cca42..814f730234 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js @@ -49,7 +49,7 @@ async function fetchPreviewImage (subjectSet, env) { subjectSet.subjects = linked.subjects } -async function buildWorkflow(workflow, displayNames, subjectSets) { +async function buildWorkflow(workflow, displayNames, subjectSets, isDefault) { const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) const workflowSubjectSets = workflow.links.subject_sets .map(subjectSetID => { @@ -61,6 +61,7 @@ async function buildWorkflow(workflow, displayNames, subjectSets) { return { completeness: workflow.completeness || 0, + default: isDefault, displayName: displayNames[workflow.id], grouped: workflow.grouped, id: workflow.id, @@ -73,9 +74,12 @@ async function fetchWorkflowsHelper (language = 'en', activeWorkflows, defaultWo const workflowIds = workflows.map(workflow => workflow.id) const displayNames = await fetchDisplayNames(language, workflowIds, env) - const awaitWorkflows = await Promise.allSettled(workflows.map(workflow => buildWorkflow(workflow, displayNames, subjectSets))) - const workflowsWithSubjectSets = awaitWorkflows.map(result => result.value) - console.log({ workflowsWithSubjectSets }) + const awaitWorkflows = workflows.map(workflow => { + const isDefault = workflows.length === 1 || workflow.id === defaultWorkflow + return buildWorkflow(workflow, displayNames, subjectSets, isDefault) + }) + const workflowStatuses = await Promise.allSettled(awaitWorkflows) + const workflowsWithSubjectSets = workflowStatuses.map(result => result.value) return workflowsWithSubjectSets } From 65f5d1c611da3c074efd53e44cf7aa178740baf3 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Mon, 8 Feb 2021 16:39:38 +0000 Subject: [PATCH 05/10] Add cellect API to tests --- .../fetchWorkflowsHelper.spec.js | 127 +++++++++++------- .../getDefaultPageProps.spec.js | 30 ++++- .../getStaticPageProps.spec.js | 30 ++++- 3 files changed, 123 insertions(+), 64 deletions(-) diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js index fb645cb8c6..fa7f5dfb27 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js @@ -2,50 +2,73 @@ import nock from 'nock' import fetchWorkflowsHelper from './fetchWorkflowsHelper' -const WORKFLOWS = [ - { - id: '1', - completeness: 0.4, - grouped: false, - links: { - subject_sets: ['1', '2', '3'] +describe('Helpers > fetchWorkflowsHelper', function () { + const WORKFLOWS = [ + { + id: '1', + completeness: 0.4, + grouped: false, + links: { + subject_sets: ['1', '2', '3'] + } + }, + { + id: '2', + completeness: 0.7, + grouped: false, + links: { + subject_sets: ['1', '2', '3'] + } } - }, - { - id: '2', - completeness: 0.7, - grouped: false, - links: { - subject_sets: ['1', '2', '3'] + ] + + // `translated_id` is a number because of a bug in the translations API :( + const TRANSLATIONS = [ + { + translated_id: 1, + strings: { + display_name: 'Foo' + } + }, + { + translated_id: 2, + strings: { + display_name: 'Bar' + } } + ] + + const availableSubjects = { + 1: 4, + 2: 10, + 3: 10 } -] - -// `translated_id` is a number because of a bug in the translations API :( -const TRANSLATIONS = [ - { - translated_id: 1, - strings: { - display_name: 'Foo' - } - }, - { - translated_id: 2, - strings: { - display_name: 'Bar' + + function subjectSet(id) { + return { + id, + display_name: `test set ${id}`, + set_member_subjects_count: 10 } } -] -function subjectSet(id) { - return { - id, - display_name: `test set ${id}`, - set_member_subjects_count: 10 - } -} + before(function () { + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/1/status') + .reply(200, { + groups: availableSubjects + }) + .get('/workflows/2/status') + .reply(200, { + groups: availableSubjects + }) + }) + + after(function () { + nock.cleanAll() + }) -describe('Helpers > fetchWorkflowsHelper', function () { it('should provide the expected result with a single workflow', async function () { const scope = nock('https://panoptes-staging.zooniverse.org/api') .get('/translations') @@ -76,9 +99,9 @@ describe('Helpers > fetchWorkflowsHelper', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) @@ -113,9 +136,9 @@ describe('Helpers > fetchWorkflowsHelper', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] }, { @@ -125,9 +148,9 @@ describe('Helpers > fetchWorkflowsHelper', function () { id: '2', displayName: 'Bar', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) @@ -163,9 +186,9 @@ describe('Helpers > fetchWorkflowsHelper', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] }, { @@ -175,9 +198,9 @@ describe('Helpers > fetchWorkflowsHelper', function () { id: '2', displayName: 'Bar', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) diff --git a/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js b/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js index 10daffc8c1..f85aa7627e 100644 --- a/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js +++ b/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js @@ -29,6 +29,12 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { } } + const availableSubjects = { + 1: 4, + 2: 10, + 3: 10 + } + function subjectSet(id) { return { id, @@ -40,6 +46,12 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { describe('with the staging API', function () { before(function () { const slug = 'test-owner/test-project' + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/1/status') + .reply(200, { + groups: availableSubjects + }) const scope = nock('https://panoptes-staging.zooniverse.org/api') .persist() .get('/projects') @@ -111,9 +123,9 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) @@ -199,6 +211,12 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { describe('with the production API', function () { before(function () { const slug = 'test-owner/test-project' + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/1/status') + .reply(200, { + groups: availableSubjects + }) const scope = nock('https://www.zooniverse.org/api') .persist() .get('/projects') @@ -270,9 +288,9 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) diff --git a/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js b/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js index cf8dadd496..23320abcd7 100644 --- a/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js +++ b/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js @@ -30,6 +30,12 @@ describe('Helpers > getStaticPageProps', function () { } } + const availableSubjects = { + 1: 4, + 2: 10, + 3: 10 + } + function subjectSet(id) { return { id, @@ -41,6 +47,12 @@ describe('Helpers > getStaticPageProps', function () { describe('with the staging API', function () { before(function () { const slug = 'test-owner/test-project' + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/1/status') + .reply(200, { + groups: availableSubjects + }) const scope = nock('https://panoptes-staging.zooniverse.org/api') .persist() .get('/projects') @@ -103,9 +115,9 @@ describe('Helpers > getStaticPageProps', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) @@ -165,6 +177,12 @@ describe('Helpers > getStaticPageProps', function () { describe('with the production API', function () { before(function () { const slug = 'test-owner/test-project' + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/1/status') + .reply(200, { + groups: availableSubjects + }) const scope = nock('https://www.zooniverse.org/api') .persist() .get('/projects') @@ -227,9 +245,9 @@ describe('Helpers > getStaticPageProps', function () { id: '1', displayName: 'Foo', subjectSets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) ] } ]) From 0d9e870a5c638b7b76550a892af9f28b1b95de0a Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Mon, 8 Feb 2021 16:56:06 +0000 Subject: [PATCH 06/10] Tidy up workflow display names --- .../helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js index 814f730234..d4a8581a3a 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js @@ -49,7 +49,7 @@ async function fetchPreviewImage (subjectSet, env) { subjectSet.subjects = linked.subjects } -async function buildWorkflow(workflow, displayNames, subjectSets, isDefault) { +async function buildWorkflow(workflow, displayName, subjectSets, isDefault) { const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) const workflowSubjectSets = workflow.links.subject_sets .map(subjectSetID => { @@ -62,7 +62,7 @@ async function buildWorkflow(workflow, displayNames, subjectSets, isDefault) { return { completeness: workflow.completeness || 0, default: isDefault, - displayName: displayNames[workflow.id], + displayName, grouped: workflow.grouped, id: workflow.id, subjectSets: workflowSubjectSets @@ -76,7 +76,8 @@ async function fetchWorkflowsHelper (language = 'en', activeWorkflows, defaultWo const awaitWorkflows = workflows.map(workflow => { const isDefault = workflows.length === 1 || workflow.id === defaultWorkflow - return buildWorkflow(workflow, displayNames, subjectSets, isDefault) + const displayName = displayNames[workflow.id] + return buildWorkflow(workflow, displayName, subjectSets, isDefault) }) const workflowStatuses = await Promise.allSettled(awaitWorkflows) const workflowsWithSubjectSets = workflowStatuses.map(result => result.value) From e072223b8afa4e9e12e49bafc397d4a8a5dde742 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Mon, 22 Feb 2021 11:07:44 +0000 Subject: [PATCH 07/10] Split code path for grouped workflows Only use the Cellect API for grouped workflows. Return a default set of workflow data otherwise. Remove included subject sets from the workflow request and request them separately, based on the Cellect response. Update tests to test for grouped and non-grouped workflows. Update mocks to represent Cellect responses without the grouped attribute. --- .../fetchWorkflowsHelper.js | 84 ++++-- .../fetchWorkflowsHelper.spec.js | 91 +++--- .../getDefaultPageProps.spec.js | 278 +++++++++++------- .../getStaticPageProps.spec.js | 274 ++++++++++------- 4 files changed, 441 insertions(+), 286 deletions(-) diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js index d4a8581a3a..0b6488ffd3 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js @@ -2,18 +2,31 @@ import { panoptes } from '@zooniverse/panoptes-js' import fetch from 'node-fetch' async function fetchWorkflowData (activeWorkflows, env) { - const response = await panoptes - .get('/workflows', { - complete: false, + const query = { + complete: false, + env, + fields: 'completeness,display_name,grouped', + id: activeWorkflows.join(',') + } + const response = await panoptes.get('/workflows', query) + const { workflows } = response.body + return workflows +} + +async function fetchSubjectSetData(subjectSetIDs, env) { + let subject_sets = [] + try { + const query = { env, - fields: 'completeness,grouped', - id: activeWorkflows.join(','), - include: 'subject_sets' - }) - const { workflows, linked } = response.body - const subjectSets = linked ? linked.subject_sets : [] - await Promise.allSettled(subjectSets.map(subjectSet => fetchPreviewImage(subjectSet, env))) - return { subjectSets, workflows } + id: subjectSetIDs.join(',') + } + const response = await panoptes.get('/subject_sets', query) + subject_sets = response.body.subject_sets + await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env))) + } catch (error) { + console.error(error) + } + return subject_sets } function fetchDisplayNames (language, activeWorkflows, env) { @@ -30,10 +43,17 @@ function fetchDisplayNames (language, activeWorkflows, env) { } async function fetchWorkflowCellectStatus(workflow) { - const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status` - const response = await fetch(workflowURL) - const body = await response.json() - const { groups } = body ?? {} + let groups = {} + if (workflow.grouped) { + try { + const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status` + const response = await fetch(workflowURL) + const body = await response.json() + groups = body.groups ?? {} + } catch (error) { + console.error(error) + } + } return groups } @@ -49,38 +69,44 @@ async function fetchPreviewImage (subjectSet, env) { subjectSet.subjects = linked.subjects } -async function buildWorkflow(workflow, displayName, subjectSets, isDefault) { +async function workflowSubjectSets(workflow, env) { const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) - const workflowSubjectSets = workflow.links.subject_sets - .map(subjectSetID => { - const subjectSet = subjectSets.find(subjectSet => subjectSet.id === subjectSetID) - subjectSet.availableSubjects = subjectSetCounts[subjectSetID] - return subjectSet - }) - .filter(Boolean) + const subjectSetIDs = Object.keys(subjectSetCounts) + const subjectSets = await fetchSubjectSetData(subjectSetIDs, env) + subjectSets.forEach(subjectSet => { + subjectSet.availableSubjects = subjectSetCounts[subjectSet.id] + }) + return subjectSets +} - return { +async function buildWorkflow(workflow, displayName, isDefault, env) { + const workflowData = { completeness: workflow.completeness || 0, default: isDefault, displayName, grouped: workflow.grouped, id: workflow.id, - subjectSets: workflowSubjectSets + subjectSets: [] + } + if (workflow.grouped) { + workflowData.subjectSets = await workflowSubjectSets(workflow, env) } + + return workflowData } async function fetchWorkflowsHelper (language = 'en', activeWorkflows, defaultWorkflow, env) { - const { subjectSets, workflows } = await fetchWorkflowData(activeWorkflows, env) + const workflows = await fetchWorkflowData(activeWorkflows, env) const workflowIds = workflows.map(workflow => workflow.id) const displayNames = await fetchDisplayNames(language, workflowIds, env) const awaitWorkflows = workflows.map(workflow => { const isDefault = workflows.length === 1 || workflow.id === defaultWorkflow - const displayName = displayNames[workflow.id] - return buildWorkflow(workflow, displayName, subjectSets, isDefault) + const displayName = displayNames[workflow.id] || workflow.display_name + return buildWorkflow(workflow, displayName, isDefault, env) }) const workflowStatuses = await Promise.allSettled(awaitWorkflows) - const workflowsWithSubjectSets = workflowStatuses.map(result => result.value) + const workflowsWithSubjectSets = workflowStatuses.map(result => result.value || result.reason) return workflowsWithSubjectSets } diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js index fa7f5dfb27..5271be2ced 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js @@ -15,7 +15,7 @@ describe('Helpers > fetchWorkflowsHelper', function () { { id: '2', completeness: 0.7, - grouped: false, + grouped: true, links: { subject_sets: ['1', '2', '3'] } @@ -56,9 +56,7 @@ describe('Helpers > fetchWorkflowsHelper', function () { const cellect = nock('https://cellect.zooniverse.org') .persist() .get('/workflows/1/status') - .reply(200, { - groups: availableSubjects - }) + .reply(200, {}) .get('/workflows/2/status') .reply(200, { groups: availableSubjects @@ -79,14 +77,16 @@ describe('Helpers > fetchWorkflowsHelper', function () { .get('/workflows') .query(true) .reply(200, { - workflows: WORKFLOWS.slice(0, 1), - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } + workflows: WORKFLOWS.slice(0, 1) + }) + .get('/subject_sets') + .query(query => query.id === '1,2,3') + .reply(200, { + subject_sets: [ + subjectSet('1'), + subjectSet('2'), + subjectSet('3') + ] }) const result = await fetchWorkflowsHelper('en', ['1']) @@ -98,11 +98,7 @@ describe('Helpers > fetchWorkflowsHelper', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] } ]) }) @@ -117,14 +113,16 @@ describe('Helpers > fetchWorkflowsHelper', function () { .get('/workflows') .query(true) .reply(200, { - workflows: WORKFLOWS, - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } + workflows: WORKFLOWS + }) + .get('/subject_sets') + .query(true) + .reply(200, { + subject_sets: [ + subjectSet('1'), + subjectSet('2'), + subjectSet('3') + ] }) const result = await fetchWorkflowsHelper('en', ['1', '2']) @@ -135,16 +133,12 @@ describe('Helpers > fetchWorkflowsHelper', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] }, { completeness: 0.7, default: false, - grouped: false, + grouped: true, id: '2', displayName: 'Bar', subjectSets: [ @@ -167,14 +161,16 @@ describe('Helpers > fetchWorkflowsHelper', function () { .get('/workflows') .query(true) .reply(200, { - workflows: WORKFLOWS, - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } + workflows: WORKFLOWS + }) + .get('/subject_sets') + .query(true) + .reply(200, { + subject_sets: [ + subjectSet('1'), + subjectSet('2'), + subjectSet('3') + ] }) const result = await fetchWorkflowsHelper('en', ['1', '2'], '2') @@ -185,16 +181,12 @@ describe('Helpers > fetchWorkflowsHelper', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] }, { completeness: 0.7, default: true, - grouped: false, + grouped: true, id: '2', displayName: 'Bar', subjectSets: [ @@ -238,14 +230,7 @@ describe('Helpers > fetchWorkflowsHelper', function () { .get('/workflows') .query(true) .reply(200, { - workflows: WORKFLOWS, - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } + workflows: WORKFLOWS }) try { diff --git a/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js b/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js index f85aa7627e..64ad08c086 100644 --- a/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js +++ b/packages/app-project/src/helpers/getDefaultPageProps/getDefaultPageProps.spec.js @@ -13,6 +13,16 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { } } + const GROUPED_PROJECT = { + id: '2', + default_workflow: '2', + primary_language: 'en', + slug: 'test-owner/grouped-project', + links: { + active_workflows: ['2'] + } + } + const TRANSLATION = { translated_id: 1, strings: { @@ -20,6 +30,13 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { } } + const GROUPED_TRANSLATION = { + translated_id: 2, + strings: { + display_name: 'Bar' + } + } + const WORKFLOW = { id: '1', completeness: 0.4, @@ -29,6 +46,15 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { } } + const GROUPED_WORKFLOW = { + id: '2', + completeness: 0.4, + grouped: true, + links: { + subject_sets: ['1', '2', '3'] + } + } + const availableSubjects = { 1: 4, 2: 10, @@ -43,53 +69,77 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { } } - describe('with the staging API', function () { - before(function () { - const slug = 'test-owner/test-project' - const cellect = nock('https://cellect.zooniverse.org') + function mockAPI(panoptesHost) { + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/2/status') + .reply(200, { + groups: availableSubjects + }) + const scope = nock(panoptesHost) .persist() - .get('/workflows/1/status') + .get('/projects') + .query(query => query.slug === 'test-owner/test-project') .reply(200, { - groups: availableSubjects + projects: [PROJECT] }) - const scope = nock('https://panoptes-staging.zooniverse.org/api') - .persist() - .get('/projects') - .query(query => query.slug === slug) - .reply(200, { - projects: [PROJECT] - }) - .get('/projects') - .query(query => query.slug !== slug) - .reply(200, { - projects: [] - }) - .get('/translations') - .query(query => { - return query.translated_type === 'workflow' - && query.translated_id === '1' - && query.language === 'en' - }) - .reply(200, { - translations: [TRANSLATION] - }) - .get('/workflows') - .query(query => query.id === '1') - .reply(200, { - workflows: [WORKFLOW], - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } - }) - .get('/workflows') - .query(query => query.id !== '1') - .reply(200, { - workflows: [] - }) + .get('/projects') + .query(query => query.slug === 'test-owner/grouped-project') + .reply(200, { + projects: [GROUPED_PROJECT] + }) + .get('/projects') + .query(query => query.slug === 'test-owner/test-wrong-project') + .reply(200, { + projects: [] + }) + .get('/translations') + .query(query => { + return query.translated_type === 'workflow' + && query.translated_id === '1' + && query.language === 'en' + }) + .reply(200, { + translations: [TRANSLATION] + }) + .get('/translations') + .query(query => { + return query.translated_type === 'workflow' + && query.translated_id === '2' + && query.language === 'en' + }) + .reply(200, { + translations: [GROUPED_TRANSLATION] + }) + .get('/subject_sets') + .query(query => query.id === '1,2,3') + .reply(200, { + subject_sets: [ + subjectSet('1'), + subjectSet('2'), + subjectSet('3') + ] + }) + .get('/workflows') + .query(query => query.id === '1') + .reply(200, { + workflows: [WORKFLOW] + }) + .get('/workflows') + .query(query => query.id === '2') + .reply(200, { + workflows: [GROUPED_WORKFLOW] + }) + .get('/workflows') + .query(query => parseInt(query.id) > 2) + .reply(200, { + workflows: [] + }) + } + + describe('with the staging API', function () { + before(function () { + mockAPI('https://panoptes-staging.zooniverse.org/api') }) after(function () { @@ -122,11 +172,7 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] } ]) }) @@ -169,6 +215,43 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { }) }) + describe('with a grouped workflow', function () { + it('should return the project\'s active workflows with subject sets', async function () { + const params = { + owner: 'test-owner', + project: 'grouped-project', + workflowID: '2' + } + const query = { + env: 'staging' + } + const req = { + connection: { + encrypted: true + }, + headers: { + host: 'www.zooniverse.org' + } + } + const res = {} + const { props } = await getDefaultPageProps({ params, query, req, res }) + expect(props.workflows).to.deep.equal([ + { + completeness: 0.4, + default: true, + grouped: true, + id: '2', + displayName: 'Bar', + subjectSets: [ + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) + ] + } + ]) + }) + }) + describe('with an invalid workflow ID', function () { let props let res = {} @@ -177,7 +260,7 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { const params = { owner: 'test-owner', project: 'test-project', - workflowID: '2' + workflowID: '3' } const query = { env: 'staging' @@ -203,58 +286,14 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { }) it('should pass an error message to the error page', function () { - expect(props.title).to.equal('Workflow 2 was not found') + expect(props.title).to.equal('Workflow 3 was not found') }) }) }) describe('with the production API', function () { before(function () { - const slug = 'test-owner/test-project' - const cellect = nock('https://cellect.zooniverse.org') - .persist() - .get('/workflows/1/status') - .reply(200, { - groups: availableSubjects - }) - const scope = nock('https://www.zooniverse.org/api') - .persist() - .get('/projects') - .query(query => query.slug === slug) - .reply(200, { - projects: [PROJECT] - }) - .get('/projects') - .query(query => query.slug !== slug) - .reply(200, { - projects: [] - }) - .get('/translations') - .query(query => { - return query.translated_type === 'workflow' - && query.translated_id === '1' - && query.language === 'en' - }) - .reply(200, { - translations: [TRANSLATION] - }) - .get('/workflows') - .query(query => query.id === '1') - .reply(200, { - workflows: [WORKFLOW], - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } - }) - .get('/workflows') - .query(query => query.id !== '1') - .reply(200, { - workflows: [] - }) + mockAPI('https://www.zooniverse.org/api') }) after(function () { @@ -287,11 +326,7 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] } ]) }) @@ -334,6 +369,43 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { }) }) + describe('with a grouped workflow', function () { + it('should return the project\'s active workflows with subject sets', async function () { + const params = { + owner: 'test-owner', + project: 'grouped-project', + workflowID: '2' + } + const query = { + env: 'production' + } + const req = { + connection: { + encrypted: true + }, + headers: { + host: 'www.zooniverse.org' + } + } + const res = {} + const { props } = await getDefaultPageProps({ params, query, req, res }) + expect(props.workflows).to.deep.equal([ + { + completeness: 0.4, + default: true, + grouped: true, + id: '2', + displayName: 'Bar', + subjectSets: [ + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) + ] + } + ]) + }) + }) + describe('with an invalid workflow ID', function () { let props let res = {} @@ -342,7 +414,7 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { const params = { owner: 'test-owner', project: 'test-project', - workflowID: '2' + workflowID: '3' } const query = { env: 'production' @@ -368,7 +440,7 @@ describe('Components > ProjectHomePage > getDefaultPageProps', function () { }) it('should pass an error message to the error page', function () { - expect(props.title).to.equal('Workflow 2 was not found') + expect(props.title).to.equal('Workflow 3 was not found') }) }) }) diff --git a/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js b/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js index 23320abcd7..cf1601ef54 100644 --- a/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js +++ b/packages/app-project/src/helpers/getStaticPageProps/getStaticPageProps.spec.js @@ -14,6 +14,16 @@ describe('Helpers > getStaticPageProps', function () { } } + const GROUPED_PROJECT = { + id: '2', + default_workflow: '2', + primary_language: 'en', + slug: 'test-owner/grouped-project', + links: { + active_workflows: ['2'] + } + } + const TRANSLATION = { translated_id: 1, strings: { @@ -21,6 +31,13 @@ describe('Helpers > getStaticPageProps', function () { } } + const GROUPED_TRANSLATION = { + translated_id: 2, + strings: { + display_name: 'Bar' + } + } + const WORKFLOW = { id: '1', completeness: 0.4, @@ -30,6 +47,15 @@ describe('Helpers > getStaticPageProps', function () { } } + const GROUPED_WORKFLOW = { + id: '2', + completeness: 0.4, + grouped: true, + links: { + subject_sets: ['1', '2', '3'] + } + } + const availableSubjects = { 1: 4, 2: 10, @@ -44,53 +70,77 @@ describe('Helpers > getStaticPageProps', function () { } } - describe('with the staging API', function () { - before(function () { - const slug = 'test-owner/test-project' - const cellect = nock('https://cellect.zooniverse.org') + function mockAPI(panoptesHost) { + const cellect = nock('https://cellect.zooniverse.org') + .persist() + .get('/workflows/2/status') + .reply(200, { + groups: availableSubjects + }) + const scope = nock(panoptesHost) .persist() - .get('/workflows/1/status') + .get('/projects') + .query(query => query.slug === 'test-owner/test-project') .reply(200, { - groups: availableSubjects + projects: [PROJECT] }) - const scope = nock('https://panoptes-staging.zooniverse.org/api') - .persist() - .get('/projects') - .query(query => query.slug === slug) - .reply(200, { - projects: [PROJECT] - }) - .get('/projects') - .query(query => query.slug !== slug) - .reply(200, { - projects: [] - }) - .get('/translations') - .query(query => { - return query.translated_type === 'workflow' - && query.translated_id === '1' - && query.language === 'en' - }) - .reply(200, { - translations: [TRANSLATION] - }) - .get('/workflows') - .query(query => query.id === '1') - .reply(200, { - workflows: [WORKFLOW], - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } - }) - .get('/workflows') - .query(query => query.id !== '1') - .reply(200, { - workflows: [] - }) + .get('/projects') + .query(query => query.slug === 'test-owner/grouped-project') + .reply(200, { + projects: [GROUPED_PROJECT] + }) + .get('/projects') + .query(query => query.slug === 'test-owner/test-wrong-project') + .reply(200, { + projects: [] + }) + .get('/translations') + .query(query => { + return query.translated_type === 'workflow' + && query.translated_id === '1' + && query.language === 'en' + }) + .reply(200, { + translations: [TRANSLATION] + }) + .get('/translations') + .query(query => { + return query.translated_type === 'workflow' + && query.translated_id === '2' + && query.language === 'en' + }) + .reply(200, { + translations: [GROUPED_TRANSLATION] + }) + .get('/subject_sets') + .query(query => query.id === '1,2,3') + .reply(200, { + subject_sets: [ + subjectSet('1'), + subjectSet('2'), + subjectSet('3') + ] + }) + .get('/workflows') + .query(query => query.id === '1') + .reply(200, { + workflows: [WORKFLOW] + }) + .get('/workflows') + .query(query => query.id === '2') + .reply(200, { + workflows: [GROUPED_WORKFLOW] + }) + .get('/workflows') + .query(query => parseInt(query.id) > 2) + .reply(200, { + workflows: [] + }) + } + + describe('with the staging API', function () { + before(function () { + mockAPI('https://panoptes-staging.zooniverse.org/api') }) after(function () { @@ -114,11 +164,7 @@ describe('Helpers > getStaticPageProps', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] } ]) }) @@ -148,6 +194,43 @@ describe('Helpers > getStaticPageProps', function () { }) }) + describe('with a grouped workflow', function () { + it('should return the project\'s active workflows with subject sets', async function () { + const params = { + owner: 'test-owner', + project: 'grouped-project', + workflowID: '2' + } + const query = { + env: 'staging' + } + const req = { + connection: { + encrypted: true + }, + headers: { + host: 'www.zooniverse.org' + } + } + const res = {} + const { props } = await getStaticPageProps({ params, query }) + expect(props.workflows).to.deep.equal([ + { + completeness: 0.4, + default: true, + grouped: true, + id: '2', + displayName: 'Bar', + subjectSets: [ + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) + ] + } + ]) + }) + }) + describe('with an invalid workflow ID', function () { let props @@ -155,7 +238,7 @@ describe('Helpers > getStaticPageProps', function () { const params = { owner: 'test-owner', project: 'test-project', - workflowID: '2' + workflowID: '3' } const query = { env: 'staging' @@ -169,58 +252,14 @@ describe('Helpers > getStaticPageProps', function () { }) it('should return a workflow error message', function () { - expect(props.title).to.equal('Workflow 2 was not found') + expect(props.title).to.equal('Workflow 3 was not found') }) }) }) describe('with the production API', function () { before(function () { - const slug = 'test-owner/test-project' - const cellect = nock('https://cellect.zooniverse.org') - .persist() - .get('/workflows/1/status') - .reply(200, { - groups: availableSubjects - }) - const scope = nock('https://www.zooniverse.org/api') - .persist() - .get('/projects') - .query(query => query.slug === slug) - .reply(200, { - projects: [PROJECT] - }) - .get('/projects') - .query(query => query.slug !== slug) - .reply(200, { - projects: [] - }) - .get('/translations') - .query(query => { - return query.translated_type === 'workflow' - && query.translated_id === '1' - && query.language === 'en' - }) - .reply(200, { - translations: [TRANSLATION] - }) - .get('/workflows') - .query(query => query.id === '1') - .reply(200, { - workflows: [WORKFLOW], - linked: { - subject_sets: [ - subjectSet('1'), - subjectSet('2'), - subjectSet('3') - ] - } - }) - .get('/workflows') - .query(query => query.id !== '1') - .reply(200, { - workflows: [] - }) + mockAPI('https://www.zooniverse.org/api') }) after(function () { @@ -244,11 +283,7 @@ describe('Helpers > getStaticPageProps', function () { grouped: false, id: '1', displayName: 'Foo', - subjectSets: [ - Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), - Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), - Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) - ] + subjectSets: [] } ]) }) @@ -286,6 +321,43 @@ describe('Helpers > getStaticPageProps', function () { }) }) + describe('with a grouped workflow', function () { + it('should return the project\'s active workflows with subject sets', async function () { + const params = { + owner: 'test-owner', + project: 'grouped-project', + workflowID: '2' + } + const query = { + env: 'production' + } + const req = { + connection: { + encrypted: true + }, + headers: { + host: 'www.zooniverse.org' + } + } + const res = {} + const { props } = await getStaticPageProps({ params, query }) + expect(props.workflows).to.deep.equal([ + { + completeness: 0.4, + default: true, + grouped: true, + id: '2', + displayName: 'Bar', + subjectSets: [ + Object.assign(subjectSet('1'), { availableSubjects: availableSubjects[1]}), + Object.assign(subjectSet('2'), { availableSubjects: availableSubjects[2]}), + Object.assign(subjectSet('3'), { availableSubjects: availableSubjects[3]}) + ] + } + ]) + }) + }) + describe('with an invalid workflow ID', function () { let props From 915d5d5675c4ef684bf253b1c83ce3d3e6f404b7 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Mon, 22 Feb 2021 13:21:27 +0000 Subject: [PATCH 08/10] log page props errors to Sentry Wrap workflow API calls in try/catch and log Node errors to Sentry. --- .../fetchWorkflowsHelper.js | 40 +++++++++++++------ .../fetchWorkflowsHelper.spec.js | 28 ++++++++----- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js index 0b6488ffd3..af23a351a2 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js @@ -1,16 +1,22 @@ import { panoptes } from '@zooniverse/panoptes-js' import fetch from 'node-fetch' +import { logToSentry } from '@helpers/logger' + async function fetchWorkflowData (activeWorkflows, env) { - const query = { - complete: false, - env, - fields: 'completeness,display_name,grouped', - id: activeWorkflows.join(',') + try { + const query = { + complete: false, + env, + fields: 'completeness,display_name,grouped', + id: activeWorkflows.join(',') + } + const response = await panoptes.get('/workflows', query) + return response.body.workflows + } catch (error) { + logToSentry(error) + throw error } - const response = await panoptes.get('/workflows', query) - const { workflows } = response.body - return workflows } async function fetchSubjectSetData(subjectSetIDs, env) { @@ -25,21 +31,28 @@ async function fetchSubjectSetData(subjectSetIDs, env) { await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env))) } catch (error) { console.error(error) + logToSentry(error) } return subject_sets } -function fetchDisplayNames (language, activeWorkflows, env) { - return panoptes - .get('/translations', { +async function fetchDisplayNames (language, activeWorkflows, env) { + let displayNames = {} + try { + const response = await panoptes.get('/translations', { env, fields: 'strings,translated_id', language, 'translated_id': activeWorkflows.join(','), 'translated_type': 'workflow' }) - .then(response => response.body.translations) - .then(createDisplayNamesMap) + const { translations } = response.body + displayNames = createDisplayNamesMap(translations) + } catch (error) { + logToSentry(error) + throw error + } + return displayNames } async function fetchWorkflowCellectStatus(workflow) { @@ -52,6 +65,7 @@ async function fetchWorkflowCellectStatus(workflow) { groups = body.groups ?? {} } catch (error) { console.error(error) + logToSentry(error) } } return groups diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js index 5271be2ced..7cb8e71ee0 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.spec.js @@ -219,29 +219,37 @@ describe('Helpers > fetchWorkflowsHelper', function () { }) describe(`when there's an error`, function () { + let workflows + it('should allow the error to be thrown for the consumer to handle', async function () { - const error = { - message: 'oh dear. oh dear god' - } + let thrownError + const mockError = new Error('oh dear. oh dear god') const scope = nock('https://panoptes-staging.zooniverse.org/api') .get('/translations') .query(true) - .replyWithError(error) + .replyWithError(mockError) .get('/workflows') .query(true) .reply(200, { workflows: WORKFLOWS }) + .get('/subject_sets') + .query(true) + .reply(200, { + subject_sets: [ + subjectSet('1'), + subjectSet('2'), + subjectSet('3') + ] + }) try { - await fetchWorkflowsHelper('en', ['1', '2'], '2') - expect.fail() + workflows = await fetchWorkflowsHelper('en', ['1', '2'], '2') } catch (error) { - expect(error).to.deep.equal({ - ...error, - response: undefined - }) + thrownError = error } + expect(thrownError).to.deep.equal(mockError) + expect(workflows).to.be.undefined() }) }) }) From ff47add17eef65a972e70121222bfdee64b595a4 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Fri, 19 Mar 2021 14:31:02 +0000 Subject: [PATCH 09/10] Split up helper files Split out the subject set data fetching into `helpers/fetchSubjectSets`. --- .../fetchSubjectSets/fetchSubjectSets.js | 65 +++++++++++++++++++ .../src/helpers/fetchSubjectSets/index.js | 1 + .../fetchWorkflowsHelper.js | 59 +---------------- 3 files changed, 68 insertions(+), 57 deletions(-) create mode 100644 packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js create mode 100644 packages/app-project/src/helpers/fetchSubjectSets/index.js diff --git a/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js b/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js new file mode 100644 index 0000000000..2097ca6358 --- /dev/null +++ b/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js @@ -0,0 +1,65 @@ +import { panoptes } from '@zooniverse/panoptes-js' +import fetch from 'node-fetch' + +import { logToSentry } from '@helpers/logger' + +async function fetchSubjectSetData(subjectSetIDs, env) { + let subject_sets = [] + try { + const query = { + env, + id: subjectSetIDs.join(',') + } + const response = await panoptes.get('/subject_sets', query) + subject_sets = response.body.subject_sets + await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env))) + } catch (error) { + console.error(error) + logToSentry(error) + } + return subject_sets +} + +async function fetchWorkflowCellectStatus(workflow) { + let groups = {} + if (workflow.grouped) { + try { + const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status` + const response = await fetch(workflowURL) + const body = await response.json() + groups = body.groups ?? {} + } catch (error) { + console.error(error) + logToSentry(error) + } + } + return groups +} + +async function fetchPreviewImage (subjectSet, env) { + try { + const response = await panoptes + .get('/set_member_subjects', { + env, + subject_set_id: subjectSet.id, + include: 'subject', + page_size: 1 + }) + const { linked } = response.body + subjectSet.subjects = linked.subjects + } catch (error) { + console.error(error) + logToSentry(error) + } +} + +export default async function workflowSubjectSets(workflow, env) { + const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) + const subjectSetIDs = Object.keys(subjectSetCounts) + const subjectSets = await fetchSubjectSetData(subjectSetIDs, env) + subjectSets.forEach(subjectSet => { + subjectSet.availableSubjects = subjectSetCounts[subjectSet.id] + }) + return subjectSets +} + diff --git a/packages/app-project/src/helpers/fetchSubjectSets/index.js b/packages/app-project/src/helpers/fetchSubjectSets/index.js new file mode 100644 index 0000000000..61b7ffdfd1 --- /dev/null +++ b/packages/app-project/src/helpers/fetchSubjectSets/index.js @@ -0,0 +1 @@ +export { default } from './fetchSubjectSets' diff --git a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js index af23a351a2..283cc98781 100644 --- a/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js +++ b/packages/app-project/src/helpers/fetchWorkflowsHelper/fetchWorkflowsHelper.js @@ -2,6 +2,7 @@ import { panoptes } from '@zooniverse/panoptes-js' import fetch from 'node-fetch' import { logToSentry } from '@helpers/logger' +import fetchSubjectSets from '@helpers/fetchSubjectSets' async function fetchWorkflowData (activeWorkflows, env) { try { @@ -18,24 +19,6 @@ async function fetchWorkflowData (activeWorkflows, env) { throw error } } - -async function fetchSubjectSetData(subjectSetIDs, env) { - let subject_sets = [] - try { - const query = { - env, - id: subjectSetIDs.join(',') - } - const response = await panoptes.get('/subject_sets', query) - subject_sets = response.body.subject_sets - await Promise.allSettled(subject_sets.map(subjectSet => fetchPreviewImage(subjectSet, env))) - } catch (error) { - console.error(error) - logToSentry(error) - } - return subject_sets -} - async function fetchDisplayNames (language, activeWorkflows, env) { let displayNames = {} try { @@ -55,44 +38,6 @@ async function fetchDisplayNames (language, activeWorkflows, env) { return displayNames } -async function fetchWorkflowCellectStatus(workflow) { - let groups = {} - if (workflow.grouped) { - try { - const workflowURL = `https://cellect.zooniverse.org/workflows/${workflow.id}/status` - const response = await fetch(workflowURL) - const body = await response.json() - groups = body.groups ?? {} - } catch (error) { - console.error(error) - logToSentry(error) - } - } - return groups -} - -async function fetchPreviewImage (subjectSet, env) { - const response = await panoptes - .get('/set_member_subjects', { - env, - subject_set_id: subjectSet.id, - include: 'subject', - page_size: 1 - }) - const { linked } = response.body - subjectSet.subjects = linked.subjects -} - -async function workflowSubjectSets(workflow, env) { - const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) - const subjectSetIDs = Object.keys(subjectSetCounts) - const subjectSets = await fetchSubjectSetData(subjectSetIDs, env) - subjectSets.forEach(subjectSet => { - subjectSet.availableSubjects = subjectSetCounts[subjectSet.id] - }) - return subjectSets -} - async function buildWorkflow(workflow, displayName, isDefault, env) { const workflowData = { completeness: workflow.completeness || 0, @@ -103,7 +48,7 @@ async function buildWorkflow(workflow, displayName, isDefault, env) { subjectSets: [] } if (workflow.grouped) { - workflowData.subjectSets = await workflowSubjectSets(workflow, env) + workflowData.subjectSets = await fetchSubjectSets(workflow, env) } return workflowData From 93e579db50cc7a48f587b626bbd69ff21e43f131 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Fri, 19 Mar 2021 18:30:29 +0000 Subject: [PATCH 10/10] Fix typo --- .../src/helpers/fetchSubjectSets/fetchSubjectSets.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js b/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js index 2097ca6358..bb40739811 100644 --- a/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js +++ b/packages/app-project/src/helpers/fetchSubjectSets/fetchSubjectSets.js @@ -53,7 +53,7 @@ async function fetchPreviewImage (subjectSet, env) { } } -export default async function workflowSubjectSets(workflow, env) { +export default async function fetchSubjectSets(workflow, env) { const subjectSetCounts = await fetchWorkflowCellectStatus(workflow) const subjectSetIDs = Object.keys(subjectSetCounts) const subjectSets = await fetchSubjectSetData(subjectSetIDs, env) @@ -62,4 +62,3 @@ export default async function workflowSubjectSets(workflow, env) { }) return subjectSets } -