From 683ee0f63aed0815616524956c3ad2bc050644c7 Mon Sep 17 00:00:00 2001 From: Krys Wisnaskas Date: Thu, 12 Sep 2024 09:30:50 -0400 Subject: [PATCH 1/6] Test deduplication on sandbox --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d1e51caade..276daf47c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -563,7 +563,7 @@ parameters: default: "al-ttahub-3196-new-tr-views" type: string sandbox_git_branch: # change to feature branch to test deployment - default: "kw-escape-quotes-in-pr-title" + default: "kw-fix-duplicate-programs" type: string prod_new_relic_app_id: default: "877570491" From 56e3e2c2eeb666aa8e43cff1992305c5b2dc1748 Mon Sep 17 00:00:00 2001 From: Krys Wisnaskas Date: Thu, 12 Sep 2024 09:43:13 -0400 Subject: [PATCH 2/6] Update known vulnerabilities --- frontend/yarn-audit-known-issues | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/yarn-audit-known-issues b/frontend/yarn-audit-known-issues index fd6cbe7f0e..570890a5bb 100644 --- a/frontend/yarn-audit-known-issues +++ b/frontend/yarn-audit-known-issues @@ -3,6 +3,6 @@ {"type":"auditAdvisory","data":{"resolution":{"id":1097682,"path":"react-scripts>jest>jest-cli>@jest/core>jest-config>jest-runner>jest-environment-jsdom>jsdom>tough-cookie","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"4.0.0","paths":["react-scripts>jest>@jest/core>jest-config>jest-environment-jsdom>jsdom>tough-cookie","react-scripts>jest>jest-cli>@jest/core>jest-config>jest-environment-jsdom>jsdom>tough-cookie","react-scripts>jest>jest-cli>@jest/core>jest-config>jest-runner>jest-environment-jsdom>jsdom>tough-cookie"]}],"metadata":null,"vulnerable_versions":"<4.1.3","module_name":"tough-cookie","severity":"moderate","github_advisory_id":"GHSA-72xf-g2v4-qvf3","cves":["CVE-2023-26136"],"access":"public","patched_versions":">=4.1.3","cvss":{"score":6.5,"vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N"},"updated":"2024-06-21T21:33:53.000Z","recommendation":"Upgrade to version 4.1.3 or later","cwe":["CWE-1321"],"found_by":null,"deleted":null,"id":1097682,"references":"- https://nvd.nist.gov/vuln/detail/CVE-2023-26136\n- https://github.com/salesforce/tough-cookie/issues/282\n- https://github.com/salesforce/tough-cookie/commit/12d474791bb856004e858fdb1c47b7608d09cf6e\n- https://github.com/salesforce/tough-cookie/releases/tag/v4.1.3\n- https://security.snyk.io/vuln/SNYK-JS-TOUGHCOOKIE-5672873\n- https://lists.debian.org/debian-lts-announce/2023/07/msg00010.html\n- https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/3HUE6ZR5SL73KHL7XUPAOEL6SB7HUDT2\n- https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/6PVVPNSAGSDS63HQ74PJ7MZ3MU5IYNVZ\n- https://security.netapp.com/advisory/ntap-20240621-0006\n- https://github.com/advisories/GHSA-72xf-g2v4-qvf3","created":"2023-07-01T06:30:16.000Z","reported_by":null,"title":"tough-cookie Prototype Pollution vulnerability","npm_advisory_id":null,"overview":"Versions of the package tough-cookie before 4.1.3 are vulnerable to Prototype Pollution due to improper handling of Cookies when using CookieJar in `rejectPublicSuffixes=false` mode. This issue arises from the manner in which the objects are initialized.","url":"https://github.com/advisories/GHSA-72xf-g2v4-qvf3"}}} {"type":"auditAdvisory","data":{"resolution":{"id":1099520,"path":"react-scripts>webpack-dev-server>express>body-parser","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"1.20.2","paths":["react-scripts>webpack-dev-server>express>body-parser"]}],"metadata":null,"vulnerable_versions":"<1.20.3","module_name":"body-parser","severity":"high","github_advisory_id":"GHSA-qwcr-r2fm-qrc7","cves":["CVE-2024-45590"],"access":"public","patched_versions":">=1.20.3","cvss":{"score":7.5,"vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"},"updated":"2024-09-10T19:01:11.000Z","recommendation":"Upgrade to version 1.20.3 or later","cwe":["CWE-405"],"found_by":null,"deleted":null,"id":1099520,"references":"- https://github.com/expressjs/body-parser/security/advisories/GHSA-qwcr-r2fm-qrc7\n- https://github.com/expressjs/body-parser/commit/b2695c4450f06ba3b0ccf48d872a229bb41c9bce\n- https://nvd.nist.gov/vuln/detail/CVE-2024-45590\n- https://github.com/advisories/GHSA-qwcr-r2fm-qrc7","created":"2024-09-10T15:52:39.000Z","reported_by":null,"title":"body-parser vulnerable to denial of service when url encoding is enabled","npm_advisory_id":null,"overview":"### Impact\n\nbody-parser <1.20.3 is vulnerable to denial of service when url encoding is enabled. A malicious actor using a specially crafted payload could flood the server with a large number of requests, resulting in denial of service.\n\n### Patches\n\nthis issue is patched in 1.20.3\n\n### References\n","url":"https://github.com/advisories/GHSA-qwcr-r2fm-qrc7"}}} {"type":"auditAdvisory","data":{"resolution":{"id":1099525,"path":"react-scripts>webpack-dev-server>express>send","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"0.18.0","paths":["react-scripts>webpack-dev-server>express>send","react-scripts>webpack-dev-server>express>serve-static>send"]}],"metadata":null,"vulnerable_versions":"<0.19.0","module_name":"send","severity":"moderate","github_advisory_id":"GHSA-m6fv-jmcg-4jfg","cves":["CVE-2024-43799"],"access":"public","patched_versions":">=0.19.0","cvss":{"score":5,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:L/I:L/A:L"},"updated":"2024-09-10T19:42:42.000Z","recommendation":"Upgrade to version 0.19.0 or later","cwe":["CWE-79"],"found_by":null,"deleted":null,"id":1099525,"references":"- https://github.com/pillarjs/send/security/advisories/GHSA-m6fv-jmcg-4jfg\n- https://nvd.nist.gov/vuln/detail/CVE-2024-43799\n- https://github.com/pillarjs/send/commit/ae4f2989491b392ae2ef3b0015a019770ae65d35\n- https://github.com/advisories/GHSA-m6fv-jmcg-4jfg","created":"2024-09-10T19:42:41.000Z","reported_by":null,"title":"send vulnerable to template injection that can lead to XSS","npm_advisory_id":null,"overview":"### Impact\n\npassing untrusted user input - even after sanitizing it - to `SendStream.redirect()` may execute untrusted code\n\n### Patches\n\nthis issue is patched in send 0.19.0\n\n### Workarounds\n\nusers are encouraged to upgrade to the patched version of express, but otherwise can workaround this issue by making sure any untrusted inputs are safe, ideally by validating them against an explicit allowlist\n\n### Details\n\nsuccessful exploitation of this vector requires the following:\n\n1. The attacker MUST control the input to response.redirect()\n1. express MUST NOT redirect before the template appears\n1. the browser MUST NOT complete redirection before:\n1. the user MUST click on the link in the template\n","url":"https://github.com/advisories/GHSA-m6fv-jmcg-4jfg"}}} +{"type":"auditAdvisory","data":{"resolution":{"id":1099525,"path":"react-scripts>webpack-dev-server>express>serve-static>send","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"0.18.0","paths":["react-scripts>webpack-dev-server>express>send","react-scripts>webpack-dev-server>express>serve-static>send"]}],"metadata":null,"vulnerable_versions":"<0.19.0","module_name":"send","severity":"moderate","github_advisory_id":"GHSA-m6fv-jmcg-4jfg","cves":["CVE-2024-43799"],"access":"public","patched_versions":">=0.19.0","cvss":{"score":5,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:L/I:L/A:L"},"updated":"2024-09-10T19:42:42.000Z","recommendation":"Upgrade to version 0.19.0 or later","cwe":["CWE-79"],"found_by":null,"deleted":null,"id":1099525,"references":"- https://github.com/pillarjs/send/security/advisories/GHSA-m6fv-jmcg-4jfg\n- https://nvd.nist.gov/vuln/detail/CVE-2024-43799\n- https://github.com/pillarjs/send/commit/ae4f2989491b392ae2ef3b0015a019770ae65d35\n- https://github.com/advisories/GHSA-m6fv-jmcg-4jfg","created":"2024-09-10T19:42:41.000Z","reported_by":null,"title":"send vulnerable to template injection that can lead to XSS","npm_advisory_id":null,"overview":"### Impact\n\npassing untrusted user input - even after sanitizing it - to `SendStream.redirect()` may execute untrusted code\n\n### Patches\n\nthis issue is patched in send 0.19.0\n\n### Workarounds\n\nusers are encouraged to upgrade to the patched version of express, but otherwise can workaround this issue by making sure any untrusted inputs are safe, ideally by validating them against an explicit allowlist\n\n### Details\n\nsuccessful exploitation of this vector requires the following:\n\n1. The attacker MUST control the input to response.redirect()\n1. express MUST NOT redirect before the template appears\n1. the browser MUST NOT complete redirection before:\n1. the user MUST click on the link in the template\n","url":"https://github.com/advisories/GHSA-m6fv-jmcg-4jfg"}}} {"type":"auditAdvisory","data":{"resolution":{"id":1099527,"path":"react-scripts>webpack-dev-server>express>serve-static","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"1.15.0","paths":["react-scripts>webpack-dev-server>express>serve-static"]}],"metadata":null,"vulnerable_versions":"<1.16.0","module_name":"serve-static","severity":"moderate","github_advisory_id":"GHSA-cm22-4g7w-348p","cves":["CVE-2024-43800"],"access":"public","patched_versions":">=1.16.0","cvss":{"score":5,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:L/I:L/A:L"},"updated":"2024-09-10T19:42:34.000Z","recommendation":"Upgrade to version 1.16.0 or later","cwe":["CWE-79"],"found_by":null,"deleted":null,"id":1099527,"references":"- https://github.com/expressjs/serve-static/security/advisories/GHSA-cm22-4g7w-348p\n- https://nvd.nist.gov/vuln/detail/CVE-2024-43800\n- https://github.com/expressjs/serve-static/commit/0c11fad159898cdc69fd9ab63269b72468ecaf6b\n- https://github.com/expressjs/serve-static/commit/ce730896fddce1588111d9ef6fdf20896de5c6fa\n- https://github.com/advisories/GHSA-cm22-4g7w-348p","created":"2024-09-10T19:42:33.000Z","reported_by":null,"title":"serve-static vulnerable to template injection that can lead to XSS","npm_advisory_id":null,"overview":"### Impact\n\npassing untrusted user input - even after sanitizing it - to `redirect()` may execute untrusted code\n\n### Patches\n\nthis issue is patched in serve-static 1.16.0\n\n### Workarounds\n\nusers are encouraged to upgrade to the patched version of express, but otherwise can workaround this issue by making sure any untrusted inputs are safe, ideally by validating them against an explicit allowlist\n\n### Details\n\nsuccessful exploitation of this vector requires the following:\n\n1. The attacker MUST control the input to response.redirect()\n1. express MUST NOT redirect before the template appears\n1. the browser MUST NOT complete redirection before:\n1. the user MUST click on the link in the template\n","url":"https://github.com/advisories/GHSA-cm22-4g7w-348p"}}} {"type":"auditAdvisory","data":{"resolution":{"id":1099529,"path":"react-scripts>webpack-dev-server>express","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"4.19.2","paths":["react-scripts>webpack-dev-server>express"]}],"metadata":null,"vulnerable_versions":"<4.20.0","module_name":"express","severity":"moderate","github_advisory_id":"GHSA-qw6h-vgh9-j6wx","cves":["CVE-2024-43796"],"access":"public","patched_versions":">=4.20.0","cvss":{"score":5,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:L/I:L/A:L"},"updated":"2024-09-10T19:41:07.000Z","recommendation":"Upgrade to version 4.20.0 or later","cwe":["CWE-79"],"found_by":null,"deleted":null,"id":1099529,"references":"- https://github.com/expressjs/express/security/advisories/GHSA-qw6h-vgh9-j6wx\n- https://nvd.nist.gov/vuln/detail/CVE-2024-43796\n- https://github.com/expressjs/express/commit/54271f69b511fea198471e6ff3400ab805d6b553\n- https://github.com/advisories/GHSA-qw6h-vgh9-j6wx","created":"2024-09-10T19:41:04.000Z","reported_by":null,"title":"express vulnerable to XSS via response.redirect()","npm_advisory_id":null,"overview":"### Impact\n\nIn express <4.20.0, passing untrusted user input - even after sanitizing it - to `response.redirect()` may execute untrusted code\n\n### Patches\n\nthis issue is patched in express 4.20.0\n\n### Workarounds\n\nusers are encouraged to upgrade to the patched version of express, but otherwise can workaround this issue by making sure any untrusted inputs are safe, ideally by validating them against an explicit allowlist\n\n### Details\n\nsuccessful exploitation of this vector requires the following:\n\n1. The attacker MUST control the input to response.redirect()\n1. express MUST NOT redirect before the template appears\n1. the browser MUST NOT complete redirection before:\n1. the user MUST click on the link in the template\n","url":"https://github.com/advisories/GHSA-qw6h-vgh9-j6wx"}}} -{"type":"auditAdvisory","data":{"resolution":{"id":1099525,"path":"react-scripts>webpack-dev-server>express>serve-static>send","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"0.18.0","paths":["react-scripts>webpack-dev-server>express>send","react-scripts>webpack-dev-server>express>serve-static>send"]}],"metadata":null,"vulnerable_versions":"<0.19.0","module_name":"send","severity":"moderate","github_advisory_id":"GHSA-m6fv-jmcg-4jfg","cves":["CVE-2024-43799"],"access":"public","patched_versions":">=0.19.0","cvss":{"score":5,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:L/I:L/A:L"},"updated":"2024-09-10T19:42:42.000Z","recommendation":"Upgrade to version 0.19.0 or later","cwe":["CWE-79"],"found_by":null,"deleted":null,"id":1099525,"references":"- https://github.com/pillarjs/send/security/advisories/GHSA-m6fv-jmcg-4jfg\n- https://nvd.nist.gov/vuln/detail/CVE-2024-43799\n- https://github.com/pillarjs/send/commit/ae4f2989491b392ae2ef3b0015a019770ae65d35\n- https://github.com/advisories/GHSA-m6fv-jmcg-4jfg","created":"2024-09-10T19:42:41.000Z","reported_by":null,"title":"send vulnerable to template injection that can lead to XSS","npm_advisory_id":null,"overview":"### Impact\n\npassing untrusted user input - even after sanitizing it - to `SendStream.redirect()` may execute untrusted code\n\n### Patches\n\nthis issue is patched in send 0.19.0\n\n### Workarounds\n\nusers are encouraged to upgrade to the patched version of express, but otherwise can workaround this issue by making sure any untrusted inputs are safe, ideally by validating them against an explicit allowlist\n\n### Details\n\nsuccessful exploitation of this vector requires the following:\n\n1. The attacker MUST control the input to response.redirect()\n1. express MUST NOT redirect before the template appears\n1. the browser MUST NOT complete redirection before:\n1. the user MUST click on the link in the template\n","url":"https://github.com/advisories/GHSA-m6fv-jmcg-4jfg"}}} From 5b3c855b6bd9a9975e97268ee29ba62fd1d792d7 Mon Sep 17 00:00:00 2001 From: Krys Wisnaskas Date: Thu, 12 Sep 2024 12:34:31 -0400 Subject: [PATCH 3/6] disable queue close --- src/lib/queue.js | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/lib/queue.js b/src/lib/queue.js index b4b3721520..b2d08869c4 100644 --- a/src/lib/queue.js +++ b/src/lib/queue.js @@ -98,46 +98,46 @@ function removeQueueEventHandlers( // Define the handlers so they can be added and removed function handleShutdown(queue) { return () => { - auditLogger.error('Shutting down, closing queue...'); - queue.close().then(() => { - auditLogger.error('Queue closed successfully.'); - removeQueueEventHandlers(queue); - process.exit(0); - }).catch((err) => { - auditLogger.error('Failed to close the queue:', err); - removeQueueEventHandlers(queue); - process.exit(1); - }); + auditLogger.error('Shutting down, but queue closing is disabled for now...'); + // queue.close().then(() => { + // auditLogger.error('Queue closed successfully.'); + // removeQueueEventHandlers(queue); + // process.exit(0); + // }).catch((err) => { + // auditLogger.error('Failed to close the queue:', err); + // removeQueueEventHandlers(queue); + // process.exit(1); + // }); }; } function handleException(queue) { return (err) => { auditLogger.error('Uncaught exception:', err); - queue.close().then(() => { - auditLogger.error('Queue closed after uncaught exception.'); - removeQueueEventHandlers(queue); - process.exit(1); - }).catch((closeErr) => { - auditLogger.error('Failed to close the queue after uncaught exception:', closeErr); - removeQueueEventHandlers(queue); - process.exit(1); - }); + // queue.close().then(() => { + // auditLogger.error('Queue closed after uncaught exception.'); + // removeQueueEventHandlers(queue); + // process.exit(1); + // }).catch((closeErr) => { + // auditLogger.error('Failed to close the queue after uncaught exception:', closeErr); + // removeQueueEventHandlers(queue); + // process.exit(1); + // }); }; } function handleRejection(queue) { return (reason, promise) => { auditLogger.error('Unhandled rejection at:', promise, 'reason:', reason); - queue.close().then(() => { - auditLogger.error('Queue closed after unhandled rejection.'); - removeQueueEventHandlers(queue); - process.exit(1); - }).catch((closeErr) => { - auditLogger.error('Failed to close the queue after unhandled rejection:', closeErr); - removeQueueEventHandlers(queue); - process.exit(1); - }); + // queue.close().then(() => { + // auditLogger.error('Queue closed after unhandled rejection.'); + // removeQueueEventHandlers(queue); + // process.exit(1); + // }).catch((closeErr) => { + // auditLogger.error('Failed to close the queue after unhandled rejection:', closeErr); + // removeQueueEventHandlers(queue); + // process.exit(1); + // }); }; } From 6e16b0757d7cc066c2cb0716e6c237d24ad27bb1 Mon Sep 17 00:00:00 2001 From: Krys Wisnaskas Date: Thu, 12 Sep 2024 14:53:26 -0400 Subject: [PATCH 4/6] Force exit on success --- src/tools/importGrantRecipientsCLI.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/tools/importGrantRecipientsCLI.js b/src/tools/importGrantRecipientsCLI.js index df5bd7245f..839693d8c4 100644 --- a/src/tools/importGrantRecipientsCLI.js +++ b/src/tools/importGrantRecipientsCLI.js @@ -15,14 +15,17 @@ const { argv: { skipdownload } } = option('skipdownload', { .help() .alias('help', 'h'); -if (skipdownload) { - processFiles().catch((e) => { - auditLogger.error(e); - return process.exit(1); - }); -} else { - updateGrantsRecipients().catch((e) => { - auditLogger.error(e); - return process.exit(1); - }); -} + (async () => { + try { + if (skipdownload) { + await processFiles(); + } else { + await updateGrantsRecipients(); + } + auditLogger.info('Script completed successfully'); + process.exit(0); + } catch (e) { + auditLogger.error(`Error during script execution: ${e.message}`, e); + process.exit(1); + } + })(); From c33dd1c66d772f8787888e45d80b2dcd20215f2b Mon Sep 17 00:00:00 2001 From: Krys Wisnaskas Date: Thu, 12 Sep 2024 14:59:49 -0400 Subject: [PATCH 5/6] Fix lint warnings --- src/tools/importGrantRecipientsCLI.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/tools/importGrantRecipientsCLI.js b/src/tools/importGrantRecipientsCLI.js index 839693d8c4..14f5a760fe 100644 --- a/src/tools/importGrantRecipientsCLI.js +++ b/src/tools/importGrantRecipientsCLI.js @@ -15,17 +15,17 @@ const { argv: { skipdownload } } = option('skipdownload', { .help() .alias('help', 'h'); - (async () => { - try { - if (skipdownload) { - await processFiles(); - } else { - await updateGrantsRecipients(); - } - auditLogger.info('Script completed successfully'); - process.exit(0); - } catch (e) { - auditLogger.error(`Error during script execution: ${e.message}`, e); - process.exit(1); +(async () => { + try { + if (skipdownload) { + await processFiles(); + } else { + await updateGrantsRecipients(); } - })(); + auditLogger.info('Script completed successfully'); + process.exit(0); + } catch (e) { + auditLogger.error(`Error during script execution: ${e.message}`, e); + process.exit(1); + } +})(); From b86c90158dd44c7061bd84d5b8879dafc0b9aa8d Mon Sep 17 00:00:00 2001 From: Krys Wisnaskas Date: Thu, 12 Sep 2024 17:46:44 -0400 Subject: [PATCH 6/6] Deduplicate program personnel. --- src/lib/updateGrantsRecipients.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/updateGrantsRecipients.js b/src/lib/updateGrantsRecipients.js index d19af58e09..254e882dd5 100644 --- a/src/lib/updateGrantsRecipients.js +++ b/src/lib/updateGrantsRecipients.js @@ -340,8 +340,13 @@ export async function processFiles(hashSumHex) { ), }))); + // Deduplicate based on 'id' + const uniqueProgramsForDb = Array.from( + new Map(programsForDb.map((item) => [item.id, item])).values(), + ); + // Extract an array of all grant personnel to update. - const programPersonnel = programsForDb.flatMap((p) => p.programPersonnel); + const programPersonnel = uniqueProgramsForDb.flatMap((p) => p.programPersonnel); // Split grants between CDI and non-CDI grants. const cdiGrants = grantsForDb.filter((g) => g.regionId === 13); @@ -400,11 +405,6 @@ export async function processFiles(hashSumHex) { await updateCDIGrantsWithOldGrantData(cdiGrantsToLink); - // Deduplicate based on 'id' - const uniqueProgramsForDb = Array.from( - new Map(programsForDb.map((item) => [item.id, item])).values(), - ); - await Program.bulkCreate( uniqueProgramsForDb, {