From b36919d7a3097d80156ba133565dcb2d11ae4f88 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 13 Mar 2019 08:42:42 -0400 Subject: [PATCH 01/52] Add MIT license file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are open source 🎉 --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2a2c252 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 The Liturgists + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From a970b95dc3137da389519bc189a5614a50290269 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 20 Mar 2019 08:20:05 -0400 Subject: [PATCH 02/52] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 28 +++++++++++++ .github/ISSUE_TEMPLATE/feature-request.md | 48 +++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..b40f4b8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment** + - OS: + - Browser: + - Version: + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..205dcf9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,48 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + + + + +This issue needs a [Quest](https://medium.com/@trek/source-quest-ff7d227d8fed). + +The goal is to give the developer who implements this feature the proper context and +starting point so as to make it easy for anyone to get started quickly, regardless of background, +domain expertise, or familiarity with a particular part of the code. + + +## Motivation + + +## Plan of action + + +## Scope + From 995ecfc55d9adb0933c42d73f409937da136cca7 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 22 Mar 2019 17:59:10 -0400 Subject: [PATCH 03/52] Add tool to sync public podcats from Podbean RSS to Contentful --- package-lock.json | 344 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 7 +- src/import.js | 236 +++++++++++++++++++++++++++++++ 3 files changed, 580 insertions(+), 7 deletions(-) create mode 100644 src/import.js diff --git a/package-lock.json b/package-lock.json index 9087c1d..9c28a29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,15 @@ "js-tokens": "^4.0.0" } }, + "@contentful/axios": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@contentful/axios/-/axios-0.18.0.tgz", + "integrity": "sha512-4r4Ww1IJlmRolKgovLTTmdS6CsdvXYVxgXRFwWSh1x36T/0Wg9kTwdaVaApZXcv1DfYyw9RSNdxIGSwTP2/Lag==", + "requires": { + "follow-redirects": "^1.2.5", + "is-buffer": "^1.1.5" + } + }, "@sentry/core": { "version": "4.6.4", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-4.6.4.tgz", @@ -752,6 +761,45 @@ "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=" }, + "cliui": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz", + "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==", + "requires": { + "string-width": "^2.1.1", + "strip-ansi": "^4.0.0", + "wrap-ansi": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -870,6 +918,25 @@ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, + "contentful-management": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/contentful-management/-/contentful-management-5.7.0.tgz", + "integrity": "sha512-hXsu9k/nMkSI2OV0zCyVB9thFcRiz2wYesKbMbFlINuAKUbVeOc4DZJQ5/anrb7X4Bin867VHZ3XtvX0+oIcPg==", + "requires": { + "@contentful/axios": "^0.18.0", + "contentful-sdk-core": "^6.3.0", + "lodash": "^4.17.11" + } + }, + "contentful-sdk-core": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/contentful-sdk-core/-/contentful-sdk-core-6.3.0.tgz", + "integrity": "sha512-PbZn4OZO/iBk4OjEHir6pX6p4c/q3lwQ3M9pLeibaoiwT8fd3JRfeL4tL0L8M2d9mS8y3g2FRDWmBAI2+0Xlcg==", + "requires": { + "lodash": "^4.17.10", + "qs": "^6.5.2" + } + }, "cookie": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", @@ -948,6 +1015,11 @@ "ms": "^2.1.1" } }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", @@ -1134,6 +1206,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1155,6 +1232,11 @@ "once": "^1.4.0" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -1958,6 +2040,11 @@ "lodash.padstart": "^4.1.0" } }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, "get-proxy": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/get-proxy/-/get-proxy-2.1.0.tgz", @@ -2325,6 +2412,11 @@ } } }, + "invert-kv": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz", + "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==" + }, "ipaddr.js": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", @@ -2666,6 +2758,14 @@ "readable-stream": "^2.0.5" } }, + "lcid": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz", + "integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==", + "requires": { + "invert-kv": "^2.0.0" + } + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -2767,6 +2867,14 @@ } } }, + "map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "requires": { + "p-defer": "^1.0.0" + } + }, "map-cache": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", @@ -2785,6 +2893,23 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "mem": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-4.2.0.tgz", + "integrity": "sha512-5fJxa68urlY0Ir8ijatKa3eRz5lwXnRCTvo9+TbTGAuTFJOwpGcY0X05moBd0nW45965Njt4CDI2GFQoG8DvqA==", + "requires": { + "map-age-cleaner": "^0.1.1", + "mimic-fn": "^2.0.0", + "p-is-promise": "^2.0.0" + }, + "dependencies": { + "mimic-fn": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.0.0.tgz", + "integrity": "sha512-jbex9Yd/3lmICXwYT6gA/j2mNQGU48wCh/VzRd+/Y/PjYQtlg1gLMdZqvu9s/xH7qKvngxRObl56XZR609IMbA==" + } + } + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -2935,8 +3060,7 @@ "nice-try": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", - "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", - "dev": true + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "nocache": { "version": "2.0.0", @@ -3168,6 +3292,52 @@ "wordwrap": "~1.0.0" } }, + "os-locale": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", + "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==", + "requires": { + "execa": "^1.0.0", + "lcid": "^2.0.0", + "mem": "^4.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "requires": { + "pump": "^3.0.0" + } + } + } + }, "os-shim": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/os-shim/-/os-shim-0.1.3.tgz", @@ -3178,11 +3348,21 @@ "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=" + }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" }, + "p-is-promise": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.0.0.tgz", + "integrity": "sha512-pzQPhYMCAgLAKPWD2jC3Se9fEfrD9npNos0y150EeqZll7akhEgGhTW/slB6lHku8AvYGiJ+YJ5hfHKePPgFWg==" + }, "p-limit": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", @@ -3249,8 +3429,7 @@ "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" }, "path-is-absolute": { "version": "1.0.1", @@ -3363,8 +3542,7 @@ "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" }, "promise-queue": { "version": "2.2.5", @@ -3390,6 +3568,15 @@ "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -3543,6 +3730,16 @@ "resolved": "https://registry.npmjs.org/replaceall/-/replaceall-0.1.6.tgz", "integrity": "sha1-gdgax663LX9cSUKt8ml6MiBojY4=" }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, "resolve": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz", @@ -3585,6 +3782,15 @@ "glob": "^7.1.3" } }, + "rss-parser": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.7.0.tgz", + "integrity": "sha512-xN1fwjVxBO0unbrUAOIUK5MAyEaaZTpKWnPY+d3QYigIG4awtbdqxHPOLuOwsTIJFsaKC78nPxIGRJG92p86Hw==", + "requires": { + "entities": "^1.1.1", + "xml2js": "^0.4.19" + } + }, "run-async": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", @@ -3756,6 +3962,11 @@ "resolved": "https://registry.npmjs.org/serverless-http/-/serverless-http-1.9.0.tgz", "integrity": "sha512-FIwGRE8e07ezj4wKIRlagHtoCoBDkxyNuUKujLjKNzpqIVVnXnIzNw1Jh5wW+keZS0ywY3c9QWnQZTRzgpADfw==" }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, "set-value": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", @@ -4603,6 +4814,11 @@ "isexe": "^2.0.0" } }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, "widest-line": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", @@ -4646,6 +4862,15 @@ "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", "dev": true }, + "wrap-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "requires": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -4699,6 +4924,11 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==" + }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", @@ -4709,6 +4939,108 @@ "resolved": "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.34.tgz", "integrity": "sha1-0A88+ddztyQUCa6SpnQNHbGfSeY=" }, + "yargs": { + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.2.tgz", + "integrity": "sha512-WyEoxgyTD3w5XRpAQNYUB9ycVH/PQrToaTXdYXRdOXvEy1l19br+VJsc0vcO8PTGg5ro/l/GY7F/JMEBmI0BxA==", + "requires": { + "cliui": "^4.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "os-locale": "^3.1.0", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz", + "integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-try": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.1.0.tgz", + "integrity": "sha512-H2RyIJ7+A3rjkwKC2l5GGtU4H1vkxKCAGsWasNVd0Set+6i4znxbWy6/j16YDPJDWxhsgZiKAstMEP8wCdSpjA==" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "yargs-parser": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.0.0.tgz", + "integrity": "sha512-w2LXjoL8oRdRQN+hOyppuXs+V/fVAYtpcrRxZuF7Kt/Oc+Jr2uAcVntaUTNT6w5ihoWfFDpNY8CPx1QskxZ/pw==", + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "dependencies": { + "camelcase": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.2.0.tgz", + "integrity": "sha512-IXFsBS2pC+X0j0N/GE7Dm7j3bsEBp+oTpb7F50dwEVX7rf3IgwO9XatnegTsDtniKCUtEJH4fSU6Asw7uoVLfQ==" + } + } + }, "yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", diff --git a/package.json b/package.json index 5520b97..03cd63d 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "private": true, "scripts": { + "import": "node ./src/import.js", "lint": "eslint", "deploy": "sls deploy", "auth-deploy": "npm run auth-sls deploy", @@ -14,14 +15,18 @@ "aws-sdk": "^2.399.0", "axios": "^0.18.0", "body-parser": "^1.18.3", + "contentful-management": "^5.7.0", "express": "^4.16.4", "helmet": "^3.15.1", "lodash": "^4.17.11", "morgan": "^1.9.1", "patreon": "^0.4.1", + "progress": "^2.0.3", "qs": "^6.6.0", + "rss-parser": "^3.7.0", "serverless": "^1.36.3", - "serverless-http": "^1.9.0" + "serverless-http": "^1.9.0", + "yargs": "^13.2.2" }, "devDependencies": { "eslint": "^5.13.0", diff --git a/src/import.js b/src/import.js new file mode 100644 index 0000000..e66fedb --- /dev/null +++ b/src/import.js @@ -0,0 +1,236 @@ +const _ = require('lodash'); +const { createClient } = require('contentful-management'); +const Parser = require('rss-parser'); +const yargs = require('yargs'); +const ProgressBar = require('progress'); + +/* eslint-disable no-restricted-syntax */ +/* eslint-disable no-await-in-loop */ + +/* + * TODO: + * 1) Fetch all podcasts from Contentful that have a feed URL + * 2) Fetch all seasons and episodes from Contentful + * 3) Fetch RSS feeds from Podbean + * 4) Create any seasons that don't already exist in Contentful + * 5) Create any episodes that don't already exist in Contentful + */ + +async function getPodcasts(environment) { + const response = await environment.getEntries({ + content_type: 'podcast', + }); + + // just the ones with a public feed + return response.items.filter(p => _.get(p.fields, 'feedUrl')); +} + +async function getSeasons(environment, podcast) { + const response = await environment.getEntries({ + content_type: 'podcastSeason', + 'fields.podcast.sys.id': podcast.sys.id, + }); + return _.keyBy(response.items, 'fields.number.en-US'); +} + +async function getEpisodes(environment, podcast) { + const response = await environment.getEntries({ + content_type: 'podcastEpisode', + limit: 1000, + 'fields.podcast.sys.id': podcast.sys.id, + }); + return response.items; +} + +async function parseFeed(feedUrl) { + const parser = new Parser({ timeout: 5000 }); + return parser.parseURL(feedUrl); +} + +const makeLink = entry => ({ + sys: { + type: 'Link', + linkType: 'Entry', + id: entry.sys.id, + }, +}); + +const withType = (value, type = null) => { + const data = type === null ? value : { + type, + value, + }; + return { 'en-US': data }; +}; + + +async function createSeason(environment, podcast, number) { + const seasonJson = { + fields: { + number: withType(parseInt(number, 10)), + title: withType(`Season ${number}`), + podcast: withType(makeLink(podcast)), + }, + }; + const season = await environment.createEntry('podcastSeason', seasonJson); + await season.publish(); + return season; +} + +function fixDuration(duration) { + // Pad the string with '0:' if it is missing hours or minutes + const nums = duration.split(':'); + const paddedNums = Array(3 - nums.length).fill('0').concat(nums); + return paddedNums.join(':'); +} + +async function syncEpisode( + environment, podcast, seasons, episodesById, rssItem, +) { + const existingEpisode = _.get(episodesById, rssItem.guid); + + const episodeJson = { + fields: { + podbeanEpisodeId: withType(rssItem.guid), + title: withType(rssItem.title), + description: withType(( + rssItem.description + || _.get(rssItem, 'content.encoded') + || rssItem.content + )), + publishedAt: withType(rssItem.isoDate), + podcast: withType(makeLink(podcast)), + }, + }; + + function setIfPresent(key, path) { + const value = _.get(rssItem, path); + if (value) { + episodeJson.fields[key] = withType(value); + } + } + + [ + ['imageUrl', 'itunes.image.href'], + ['mediaUrl', 'enclosure.url'], + ['duration', 'itunes.duration'], + ].forEach(args => setIfPresent(...args)); + + episodeJson.fields.duration['en-US'] = fixDuration( + episodeJson.fields.duration['en-US'], + ); + + const seasonEpisodeNumber = _.get(rssItem, 'itunes.episode'); + if (seasonEpisodeNumber) { + episodeJson.fields.seasonEpisodeNumber = withType( + parseInt(seasonEpisodeNumber, 10), + ); + } + + const seasonNumber = _.get(rssItem, 'itunes.season'); + if (seasonNumber && seasons[seasonNumber]) { + episodeJson.fields.season = withType(makeLink(seasons[seasonNumber])); + } + + if (!existingEpisode) { + const episode = await environment.createEntry( + 'podcastEpisode', episodeJson, + ); + await episode.publish(); + return episode; + } + + if (!_.isEqual(existingEpisode.fields, episodeJson.fields)) { + existingEpisode.fields = episodeJson.fields; + await existingEpisode.update(); + await existingEpisode.publish(); + } + + return existingEpisode; +} + +function getSeasonNumbersFromFeed(feed) { + return new Set( + _(feed.items) + .map('itunes.season') + .filter(s => !_.isUndefined(s)) + .value(), + ); +} + +async function syncPodcast(environment, podcast) { + const feedUrl = `${podcast.fields.feedUrl['en-US']}`; + console.log(`Parsing feed from ${feedUrl}`); + const feed = await parseFeed(feedUrl); + + const seasonNumbers = getSeasonNumbersFromFeed(feed); + const seasons = await getSeasons(environment, podcast); + const newSeasonNumbers = Array.from(seasonNumbers) + .filter(sn => !_.has(seasons, sn)); + + if (newSeasonNumbers.length > 0) { + const bar = new ProgressBar( + 'Creating seasons: :current/:total :bar', + { total: newSeasonNumbers.length }, + ); + for (const seasonNumber of newSeasonNumbers) { + seasons[seasonNumber] = await createSeason( + environment, podcast, seasonNumber, + ); + bar.tick(); + } + } + + const episodes = await getEpisodes(environment, podcast); + const episodesById = _.keyBy(episodes, 'fields.podbeanEpisodeId.en-US'); + const bar = new ProgressBar( + 'Syncing episodes: :current/:total :bar', + { total: feed.items.length }, + ); + for (const item of feed.items) { + await syncEpisode( + environment, podcast, seasons, episodesById, item, + ); + bar.tick(); + } +} + +async function syncPodcasts(accessToken, spaceId, environmentId) { + const contentful = createClient({ accessToken }); + const space = await contentful.getSpace(spaceId); + const environment = await space.getEnvironment(environmentId); + + console.log('Getting public podcasts...'); + const podcasts = await getPodcasts(environment); + console.log(`Syncing ${podcasts.length} public podcasts...`); + for (const podcast of podcasts) { + await syncPodcast(environment, podcast); + } +} + +if (require.main === module) { + const { argv } = yargs + .options({ + accessToken: { + describe: 'Contentful access token', + demandOption: true, + }, + spaceId: { + describe: 'Contentful space ID', + demandOption: true, + }, + environmentId: { + describe: 'Contentful environment ID', + demandOption: true, + }, + }); + + const { accessToken, spaceId, environmentId } = argv; + + syncPodcasts(accessToken, spaceId, environmentId) + .then(() => console.log('Done.')) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} From ceabb394b903530a7009826edda92d2fd83749f7 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 25 Mar 2019 09:38:44 -0400 Subject: [PATCH 04/52] Determine (and create) contributors and tags from episode description Look for structured text and parse contributor/tag names. TODO: document format --- .eslintrc.js | 5 ++- src/import.js | 100 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 19 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 3705054..f1555f1 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,3 +1,6 @@ module.exports = { - "extends": "airbnb-base" + "extends": "airbnb-base", + "rules": { + "no-underscore-dangle": [ "error", { "allowAfterThis": true } ] + } }; diff --git a/src/import.js b/src/import.js index e66fedb..5837fb2 100644 --- a/src/import.js +++ b/src/import.js @@ -7,15 +7,6 @@ const ProgressBar = require('progress'); /* eslint-disable no-restricted-syntax */ /* eslint-disable no-await-in-loop */ -/* - * TODO: - * 1) Fetch all podcasts from Contentful that have a feed URL - * 2) Fetch all seasons and episodes from Contentful - * 3) Fetch RSS feeds from Podbean - * 4) Create any seasons that don't already exist in Contentful - * 5) Create any episodes that don't already exist in Contentful - */ - async function getPodcasts(environment) { const response = await environment.getEntries({ content_type: 'podcast', @@ -84,25 +75,96 @@ function fixDuration(duration) { return paddedNums.join(':'); } +class ContentfulCache { + constructor(environment) { + this.environment = environment; + this.cache = {}; + this.pks = { + contributor: 'name', + tag: 'name', + }; + } + + _getPk(contentType) { + return _.get(this.pks, contentType, 'title'); + } + + async _initContentType(contentType) { + const pk = this._getPk(contentType); + const data = await this.environment.getEntries({ + content_type: contentType, + }); + this.cache[contentType] = _.keyBy(data.items, `fields.${pk}.en-US`); + } + + async getOrCreate(contentType, pkValue) { + if (!_.has(this.cache, contentType)) { + await this._initContentType(contentType); + } + + if (_.has(this.cache, [contentType, pkValue])) { + return this.cache[contentType][pkValue]; + } + + const pk = this._getPk(contentType); + + const entry = await this.environment.createEntry( + contentType, + { + fields: { + [pk]: withType(pkValue), + }, + }, + ); + this.cache[contentType][pkValue] = entry; + await entry.publish(); + return entry; + } +} + async function syncEpisode( - environment, podcast, seasons, episodesById, rssItem, + environment, cache, podcast, seasons, episodesById, rssItem, ) { const existingEpisode = _.get(episodesById, rssItem.guid); + const description = ( + rssItem.description + || _.get(rssItem, 'content.encoded') + || rssItem.content + ); const episodeJson = { fields: { podbeanEpisodeId: withType(rssItem.guid), title: withType(rssItem.title), - description: withType(( - rssItem.description - || _.get(rssItem, 'content.encoded') - || rssItem.content - )), + description: withType(description), publishedAt: withType(rssItem.isoDate), podcast: withType(makeLink(podcast)), }, }; + // Grab contributors and tags from description, + // create them in Contentful if necessary, and + // set them on the episode. + const lines = description.match(/[^\r\n]+/g); + for (const line of lines) { + // look for "contributors:" or "tags:" followed by a + // comma-separated list of names. + const match = line.match( + /(contributors|tags):([^,]+(?:,[^,]+)*);/, + ); + if (match) { + const [label, valuesCsv] = match.slice(1); + const contentType = label.replace(/s$/, ''); + const values = valuesCsv.split(','); + const links = []; + for (const value of values) { + const entry = await cache.getOrCreate(contentType, value); + links.push(makeLink(entry)); + } + episodeJson.fields[label] = withType(links); + } + } + function setIfPresent(key, path) { const value = _.get(rssItem, path); if (value) { @@ -158,7 +220,7 @@ function getSeasonNumbersFromFeed(feed) { ); } -async function syncPodcast(environment, podcast) { +async function syncPodcast(environment, cache, podcast) { const feedUrl = `${podcast.fields.feedUrl['en-US']}`; console.log(`Parsing feed from ${feedUrl}`); const feed = await parseFeed(feedUrl); @@ -189,7 +251,7 @@ async function syncPodcast(environment, podcast) { ); for (const item of feed.items) { await syncEpisode( - environment, podcast, seasons, episodesById, item, + environment, cache, podcast, seasons, episodesById, item, ); bar.tick(); } @@ -200,11 +262,13 @@ async function syncPodcasts(accessToken, spaceId, environmentId) { const space = await contentful.getSpace(spaceId); const environment = await space.getEnvironment(environmentId); + const cache = new ContentfulCache(environment); + console.log('Getting public podcasts...'); const podcasts = await getPodcasts(environment); console.log(`Syncing ${podcasts.length} public podcasts...`); for (const podcast of podcasts) { - await syncPodcast(environment, podcast); + await syncPodcast(environment, cache, podcast); } } From a4356ce35c2ce4b6d4c4170326642bab2e799339 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 25 Mar 2019 21:35:58 -0400 Subject: [PATCH 05/52] Fix update bug --- src/import.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/import.js b/src/import.js index 5837fb2..5565b35 100644 --- a/src/import.js +++ b/src/import.js @@ -125,7 +125,7 @@ class ContentfulCache { async function syncEpisode( environment, cache, podcast, seasons, episodesById, rssItem, ) { - const existingEpisode = _.get(episodesById, rssItem.guid); + let existingEpisode = _.get(episodesById, rssItem.guid); const description = ( rssItem.description @@ -204,7 +204,7 @@ async function syncEpisode( if (!_.isEqual(existingEpisode.fields, episodeJson.fields)) { existingEpisode.fields = episodeJson.fields; - await existingEpisode.update(); + existingEpisode = await existingEpisode.update(); await existingEpisode.publish(); } From 3838a1e9627d49d83b8e80c83d662b0e44bccd6f Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 26 Mar 2019 10:07:53 -0400 Subject: [PATCH 06/52] Create sync lambda and schedule --- .eslintrc.js | 3 +- aws/deploy_policy.json | 74 +++++++++++++++++++++----------------- index.js | 30 ++-------------- package.json | 2 +- serverless.yml | 10 ++++++ src/creds.js | 34 ++++++++++++++++++ src/{import.js => sync.js} | 72 +++++++++++++++++++++++++++++++------ 7 files changed, 152 insertions(+), 73 deletions(-) create mode 100644 src/creds.js rename src/{import.js => sync.js} (81%) diff --git a/.eslintrc.js b/.eslintrc.js index f1555f1..1bd1001 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,6 +1,7 @@ module.exports = { "extends": "airbnb-base", "rules": { - "no-underscore-dangle": [ "error", { "allowAfterThis": true } ] + "no-underscore-dangle": [ "error", { "allowAfterThis": true } ], + "import/prefer-default-export": [ "off" ], } }; diff --git a/aws/deploy_policy.json b/aws/deploy_policy.json index acd2914..ea0cf1f 100644 --- a/aws/deploy_policy.json +++ b/aws/deploy_policy.json @@ -140,55 +140,63 @@ { "Effect": "Allow", "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListHostedZones", - "route53:ListResourceRecordSets" + "route53:ChangeResourceRecordSets", + "route53:ListHostedZones", + "route53:ListResourceRecordSets" ], "Resource": "*" }, { - "Effect": "Allow", - "Action": [ - "acm:AddTagsToCertificate", - "acm:RequestCertificate", - "acm:DescribeCertificate", - "acm:DeleteCertificate" - ], - "Resource": "*" + "Effect": "Allow", + "Action": [ + "acm:AddTagsToCertificate", + "acm:RequestCertificate", + "acm:DescribeCertificate", + "acm:DeleteCertificate" + ], + "Resource": "*" }, { - "Effect": "Allow", - "Action": [ - "apigateway:GET", - "apigateway:POST", - "apigateway:DELETE" - ], - "Resource": [ - "arn:aws:apigateway:*::/domainnames", - "arn:aws:apigateway:*::/domainnames/*" - ] + "Effect": "Allow", + "Action": [ + "apigateway:GET", + "apigateway:POST", + "apigateway:DELETE" + ], + "Resource": [ + "arn:aws:apigateway:*::/domainnames", + "arn:aws:apigateway:*::/domainnames/*" + ] }, { - "Effect": "Allow", - "Action": [ - "cloudfront:UpdateDistribution" - ], - "Resource": "*" + "Effect": "Allow", + "Action": [ + "cloudfront:UpdateDistribution" + ], + "Resource": "*" }, { "Action": [ - "ssm:DescribeParameters" + "ssm:DescribeParameters" ], "Resource": "*", "Effect": "Allow" }, { - "Action": [ - "ssm:GetParameter", - "ssm:GetParameters" - ], - "Resource": "arn:aws:ssm:us-east-1:682897274253:parameter/dev/*", - "Effect": "Allow" + "Action": [ + "ssm:GetParameter", + "ssm:GetParameters" + ], + "Resource": "arn:aws:ssm:us-east-1:682897274253:parameter/dev/*", + "Effect": "Allow" + }, + { + "Action": [ + "cloudwatch:*", + "events:*" + ], + "Resource": "*", + "Effect": "Allow" } ] } diff --git a/index.js b/index.js index db9e435..fbe9e8e 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,6 @@ const serverless = require('serverless-http'); const bodyParser = require('body-parser'); const express = require('express'); const qs = require('qs'); -const awsParamStore = require('aws-param-store'); const axios = require('axios'); const helmet = require('helmet'); const morgan = require('morgan'); @@ -10,34 +9,9 @@ const _ = require('lodash'); const Sentry = require('@sentry/node'); const { patreon: patreonAPI } = require('patreon'); -const stage = process.env.SLS_STAGE; - -const credSpecs = { - patreon: { - client_id: `/${stage}/PATREON_CLIENT_ID`, - client_secret: `/${stage}/PATREON_CLIENT_SECRET`, - campaign_url: `/${stage}/PATREON_CAMPAIGN_URL`, - }, - contentful: { - space: `/${stage}/CONTENTFUL_SPACE`, - environment: `/${stage}/CONTENTFUL_ENVIRONMENT`, - accessToken: `/${stage}/CONTENTFUL_ACCESS_TOKEN`, - }, - sentry: { - dsn: `/${stage}/SENTRY_DSN`, - }, -}; +const { getCreds } = require('./src/creds'); -async function getCreds(name) { - const credSpec = credSpecs[name]; - const opts = { region: process.env.AWS_REGION }; - const keys = _.keys(credSpec); - const params = await Promise.all( - keys.map(key => awsParamStore.getParameter(credSpec[key], opts)), - ); - const values = _.map(params, 'Value'); - return _.zipObject(keys, values); -} +const stage = process.env.SLS_STAGE; function canAccess(pledge, item, podcasts) { const contentType = item.sys.contentType.sys.id; diff --git a/package.json b/package.json index 03cd63d..dd2c148 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.0.1", "private": true, "scripts": { - "import": "node ./src/import.js", + "sync": "node ./src/sync.js", "lint": "eslint", "deploy": "sls deploy", "auth-deploy": "npm run auth-sls deploy", diff --git a/serverless.yml b/serverless.yml index b45a6b7..514abfe 100644 --- a/serverless.yml +++ b/serverless.yml @@ -32,6 +32,16 @@ functions: - http: GET /patreon/authorize - http: POST /patreon/validate - http: GET /contentful/{proxy+} + sync: + handler: src/sync.handler + timeout: 300 + events: + - schedule: + rate: cron(0 12 * * ? *) # daily, 8AM EDT + enabled: ${self:custom.syncEnabled.${self:custom.stage}} custom: stage: ${opt:stage, self:provider.stage} + syncEnabled: + dev: false + staging: true diff --git a/src/creds.js b/src/creds.js new file mode 100644 index 0000000..8156999 --- /dev/null +++ b/src/creds.js @@ -0,0 +1,34 @@ +const _ = require('lodash'); +const awsParamStore = require('aws-param-store'); + +const stage = process.env.SLS_STAGE; + +const credSpecs = { + patreon: { + client_id: `/${stage}/PATREON_CLIENT_ID`, + client_secret: `/${stage}/PATREON_CLIENT_SECRET`, + campaign_url: `/${stage}/PATREON_CAMPAIGN_URL`, + }, + contentful: { + space: `/${stage}/CONTENTFUL_SPACE`, + environment: `/${stage}/CONTENTFUL_ENVIRONMENT`, + accessToken: `/${stage}/CONTENTFUL_ACCESS_TOKEN`, + }, + contentfulManagement: { + accessToken: `/${stage}/CONTENTFUL_MANAGEMENT_ACCESS_TOKEN`, + }, + sentry: { + dsn: `/${stage}/SENTRY_DSN`, + }, +}; + +module.exports.getCreds = async (name) => { + const credSpec = credSpecs[name]; + const opts = { region: process.env.AWS_REGION }; + const keys = _.keys(credSpec); + const params = await Promise.all( + keys.map(key => awsParamStore.getParameter(credSpec[key], opts)), + ); + const values = _.map(params, 'Value'); + return _.zipObject(keys, values); +}; diff --git a/src/import.js b/src/sync.js similarity index 81% rename from src/import.js rename to src/sync.js index 5565b35..543b9c5 100644 --- a/src/import.js +++ b/src/sync.js @@ -3,6 +3,9 @@ const { createClient } = require('contentful-management'); const Parser = require('rss-parser'); const yargs = require('yargs'); const ProgressBar = require('progress'); +const Sentry = require('@sentry/node'); + +const { getCreds } = require('./creds'); /* eslint-disable no-restricted-syntax */ /* eslint-disable no-await-in-loop */ @@ -122,11 +125,20 @@ class ContentfulCache { } } +function logProgress(progress, msg) { + if (progress.stream.clearLine) { + progress.interrupt(msg); + } else { + console.log(msg); + } +} + async function syncEpisode( - environment, cache, podcast, seasons, episodesById, rssItem, + progress, environment, cache, podcast, seasons, episodesById, rssItem, ) { let existingEpisode = _.get(episodesById, rssItem.guid); + const { title } = rssItem; const description = ( rssItem.description || _.get(rssItem, 'content.encoded') @@ -135,7 +147,7 @@ async function syncEpisode( const episodeJson = { fields: { podbeanEpisodeId: withType(rssItem.guid), - title: withType(rssItem.title), + title: withType(title), description: withType(description), publishedAt: withType(rssItem.isoDate), podcast: withType(makeLink(podcast)), @@ -195,20 +207,24 @@ async function syncEpisode( } if (!existingEpisode) { + logProgress(progress, `Creating episode: "${title}"`); const episode = await environment.createEntry( 'podcastEpisode', episodeJson, ); await episode.publish(); - return episode; + return { episode, status: 'created' }; } + let status = 'unchanged'; if (!_.isEqual(existingEpisode.fields, episodeJson.fields)) { + logProgress(progress, `Updating episode: "${title}"`); + status = 'updated'; existingEpisode.fields = episodeJson.fields; existingEpisode = await existingEpisode.update(); await existingEpisode.publish(); } - return existingEpisode; + return { episode: existingEpisode, status }; } function getSeasonNumbersFromFeed(feed) { @@ -221,7 +237,8 @@ function getSeasonNumbersFromFeed(feed) { } async function syncPodcast(environment, cache, podcast) { - const feedUrl = `${podcast.fields.feedUrl['en-US']}`; + const feedUrl = podcast.fields.feedUrl['en-US']; + console.log(`Parsing feed from ${feedUrl}`); const feed = await parseFeed(feedUrl); @@ -241,19 +258,32 @@ async function syncPodcast(environment, cache, podcast) { ); bar.tick(); } + console.log(`Created ${newSeasonNumbers.length} seasons`); } const episodes = await getEpisodes(environment, podcast); const episodesById = _.keyBy(episodes, 'fields.podbeanEpisodeId.en-US'); + const counts = { + updated: 0, + created: 0, + unchanged: 0, + }; const bar = new ProgressBar( - 'Syncing episodes: :current/:total :bar', - { total: feed.items.length }, + 'Syncing episodes: [:bar] :current/:total (:created created, :updated updated)', + { + total: feed.items.length, + incomplete: ' ', + width: 20, + ...counts, + }, ); for (const item of feed.items) { - await syncEpisode( - environment, cache, podcast, seasons, episodesById, item, + const { status } = await syncEpisode( + bar, environment, cache, podcast, seasons, episodesById, item, ); - bar.tick(); + + counts[status] += 1; + bar.tick(counts); } } @@ -298,3 +328,25 @@ if (require.main === module) { process.exit(1); }); } + +module.exports.handler = async () => { + const sentry = await getCreds('sentry'); + Sentry.init(sentry); + Sentry.configureScope((scope) => { + scope.setTag('stage', process.env.SLS_STAGE); + }); + + const contentful = await getCreds('contentful'); + const contentfulManagement = await getCreds('contentfulManagement'); + return syncPodcasts( + contentfulManagement.accessToken, + contentful.space, + contentful.environment, + ) + .then(() => console.log('Done.')) + .catch((err) => { + console.error(err); + Sentry.captureException(err); + throw err; + }); +}; From 696242564d75441391d983370aaef141fd5bdbac Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 3 May 2019 21:07:00 -0400 Subject: [PATCH 07/52] Add namespace to stack name Allows for multiple dev stacks. --- serverless.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/serverless.yml b/serverless.yml index 514abfe..96f1416 100644 --- a/serverless.yml +++ b/serverless.yml @@ -1,4 +1,4 @@ -service: theliturgists-backend +service: theliturgists-backend-${self:custom.namespace} provider: name: aws @@ -41,6 +41,7 @@ functions: enabled: ${self:custom.syncEnabled.${self:custom.stage}} custom: + namespace: ${env:SLS_NAMESPACE, env:USER} stage: ${opt:stage, self:provider.stage} syncEnabled: dev: false From 08867d75023c8705f2fcfad708ba0d31d341e737 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 7 May 2019 22:38:33 -0400 Subject: [PATCH 08/52] Fliter liturgy items by patron tier --- index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index fbe9e8e..dc976dc 100644 --- a/index.js +++ b/index.js @@ -25,7 +25,10 @@ function canAccess(pledge, item, podcasts) { ) ); } - if (contentType === 'meditation') { + if (contentType === 'meditation' || contentType === 'liturgyItem') { + // Patrons with the 'Master Meditations' reward tier + // get access to both Meditations and Liturgies + // in addition to patrons-only podcasts. return ( pledge && /Meditations/i.test(pledge.reward.title) ); From 67927b533458e5b89b5552a92432f025530f4d2b Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Thu, 9 May 2019 23:10:30 -0400 Subject: [PATCH 09/52] Extract contentfulGet function for reuse --- index.js | 109 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 43 deletions(-) diff --git a/index.js b/index.js index dc976dc..07f8c76 100644 --- a/index.js +++ b/index.js @@ -146,51 +146,74 @@ async function init() { ), ); - app.get('*/contentful/*', wrapAsync( - async (req, res) => { - const contentful = await getCreds('contentful'); - const { campaign_url: patreonCampaignUrl } = await getCreds('patreon'); + /** + * Fetch contentful data, filtered by Patreon access. + * + * @param {string} path contentful resource path (after environment) + * @param {object} params contentful query params from request object + * @param {string} patreonToken patreon token from request header + * @returns {object} contentful API response + * @throws {Error} on contentful API error + * error object has 'status' and 'json' fields + */ + async function contentfulGet(path, params, patreonToken) { + const contentful = await getCreds('contentful'); + const { space, environment } = contentful; + const { campaign_url: patreonCampaignUrl } = await getCreds('patreon'); + + const fullPath = `/spaces/${space}/environments/${environment}/${path}`; + const host = 'https://cdn.contentful.com'; + const url = `${host}/${fullPath}`; + Sentry.addBreadcrumb({ message: 'Making contentful request', data: url }); + const contentfulRes = await axios.get(url, { + params, + validateStatus: null, + headers: { + authorization: `Bearer ${contentful.accessToken}`, + }, + }); + const patreon = { + token: patreonToken, + campaign_url: patreonCampaignUrl, + }; + const { status } = contentfulRes; + const { data } = contentfulRes; + Sentry.addBreadcrumb({ message: 'Got contentful response', data }); + + if (status >= 400) { + const e = new Error(); + e.status = status; + e.json = data; + throw e; + } - Sentry.addBreadcrumb({ message: 'API request', data: req.path }); - const path = req.path - .replace(new RegExp(`^(/${stage})?/contentful`), '') - .replace(/\/spaces\/[^/]+/, `/spaces/${contentful.space}`) - .replace(/\/environments\/[^/]+/, `/environments/${contentful.environment}`); - const host = 'https://cdn.contentful.com'; - const url = `${host}/${path}`; - Sentry.addBreadcrumb({ message: 'Making contentful request', data: url }); - const contentfulRes = await axios.get(url, { - params: req.query, - validateStatus: null, - headers: { - authorization: `Bearer ${contentful.accessToken}`, - }, - }); - const patreon = { - token: _.get(req.headers, 'x-theliturgists-patreon-token'), - campaign_url: patreonCampaignUrl, + try { + return await filterData(data, patreon); + } catch (e) { + e.json = { + error: ( + 'Error verifying Patreon status. ' + + 'Please re-connect Patreon and try again.' + ), }; - const { status } = contentfulRes; - let { data } = contentfulRes; - Sentry.addBreadcrumb({ message: 'Got contentful response', data }); - - let patreonError; - if (status >= 200 && status < 400) { - try { - data = await filterData(data, patreon); - } catch (e) { - patreonError = e.error; - } - } - if (patreonError) { - res.status(patreonError.status).json({ - error: ( - 'Error verifying Patreon status. ' - + 'Please re-connect Patreon and try again.' - ), - }); - } else { - res.status(status).json(data); + throw e.error; + } + } + + app.get('*/contentful/spaces/:space/environments/:env/*', wrapAsync( + async (req, res) => { + Sentry.addBreadcrumb({ message: 'API request', data: req.path }); + + const path = req.params[1]; + const params = req.query; + const patreonToken = _.get(req.headers, 'x-theliturgists-patreon-token'); + + try { + const data = await contentfulGet(path, params, patreonToken); + res.status(200).json(data); + } catch (e) { + console.log(e); + res.status(e.status).json(e.json); } }, )); From c2e771e20a1ed1257a9ee8249535732f542ffd6c Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 10 May 2019 11:58:01 -0400 Subject: [PATCH 10/52] Add RSS endpoints with patreon access control --- index.js | 135 +++++++++++++++++++++++------ package-lock.json | 214 ++++++++++++++++++++++++++++++++++------------ package.json | 3 +- serverless.yml | 3 + 4 files changed, 277 insertions(+), 78 deletions(-) diff --git a/index.js b/index.js index 07f8c76..11dddcc 100644 --- a/index.js +++ b/index.js @@ -13,40 +13,70 @@ const { getCreds } = require('./src/creds'); const stage = process.env.SLS_STAGE; +function canAccessPodcast(pledge, podcast) { + return ( + _.get(podcast.fields, 'minimumPledgeDollars', null) === null + || ( + !!pledge + && podcast.fields.minimumPledgeDollars * 100 <= pledge.amount_cents + ) + ); +} + +function canAccessPatronMedia(pledge) { + // Patrons with the 'Master Meditations' reward tier + // get access to both Meditations and Liturgies + // in addition to patrons-only podcasts. + return pledge && /Meditations/i.test(pledge.reward.title); +} + function canAccess(pledge, item, podcasts) { const contentType = item.sys.contentType.sys.id; if (contentType === 'podcastEpisode') { const podcast = podcasts[item.fields.podcast.sys.id]; - return ( - _.get(podcast.fields, 'minimumPledgeDollars', null) === null - || ( - !!pledge - && podcast.fields.minimumPledgeDollars * 100 <= pledge.amount_cents - ) - ); + return canAccessPodcast(pledge, podcast); } if (contentType === 'meditation' || contentType === 'liturgyItem') { - // Patrons with the 'Master Meditations' reward tier - // get access to both Meditations and Liturgies - // in addition to patrons-only podcasts. - return ( - pledge && /Meditations/i.test(pledge.reward.title) - ); + return canAccessPatronMedia(pledge); } return true; } -async function filterData(contentfulData, patreon) { - let pledge = null; +async function getPledge(patreon) { + const { token, campaignUrl } = patreon; + if (!token || !campaignUrl) { + return null; + } - if (patreon.token) { - const client = patreonAPI(patreon.token); - const { store, rawJson } = await client('/current_user?includes=pledges'); - const user = store.find('user', rawJson.data.id); + const client = patreonAPI(token); + let resp; + try { + resp = await client('/current_user?includes=pledges'); + } catch (patreonError) { + try { + // patreonError.response is a response object from fetch() + const patreonResponse = await patreonError.response.json(); + console.log(`Patreon error: ${JSON.stringify(patreonResponse, null, 2)}`); + } catch (e) { + console.log(`Error retrieving Patreon error: ${e}`); + } - pledge = _.find(user.pledges, p => (p.reward.campaign.url === patreon.campaign_url)); + // no matter what the error was, deny access. + return null; } + const { store, rawJson } = resp; + const user = store.find('user', rawJson.data.id); + + return _.find( + user.pledges, + p => (p.reward.campaign.url === campaignUrl), + ); +} + +async function filterData(contentfulData, patreon) { + const pledge = getPledge(patreon); + // podcasts are included; pull them out by ID first const podcasts = _.fromPairs( contentfulData.includes.Entry.filter( @@ -156,15 +186,18 @@ async function init() { * @throws {Error} on contentful API error * error object has 'status' and 'json' fields */ - async function contentfulGet(path, params, patreonToken) { + async function contentfulGet(path, params, patreonToken, filter = true) { const contentful = await getCreds('contentful'); const { space, environment } = contentful; - const { campaign_url: patreonCampaignUrl } = await getCreds('patreon'); + const { campaign_url: campaignUrl } = await getCreds('patreon'); const fullPath = `/spaces/${space}/environments/${environment}/${path}`; const host = 'https://cdn.contentful.com'; const url = `${host}/${fullPath}`; - Sentry.addBreadcrumb({ message: 'Making contentful request', data: url }); + Sentry.addBreadcrumb({ + message: 'Making contentful request', + data: { url }, + }); const contentfulRes = await axios.get(url, { params, validateStatus: null, @@ -174,7 +207,7 @@ async function init() { }); const patreon = { token: patreonToken, - campaign_url: patreonCampaignUrl, + campaignUrl, }; const { status } = contentfulRes; const { data } = contentfulRes; @@ -187,6 +220,10 @@ async function init() { throw e; } + if (!filter) { + return data; + } + try { return await filterData(data, patreon); } catch (e) { @@ -218,6 +255,56 @@ async function init() { }, )); + async function canAccessFeed(collection, collectionId, patreon) { + const pledge = await getPledge(patreon); + if (collection === 'podcast') { + const path = `entries/${collectionId}`; + const podcast = await contentfulGet(path, {}, patreon.token, false); + return canAccessPodcast(pledge, podcast); + } + return canAccessPatronMedia(pledge); + } + + app.get('*/rss/:collection/:collectionId', wrapAsync( + async (req, res) => { + const { collection, collectionId } = req.params; + const collectionFields = { + podcast: 'podcast', + 'meditation-category': 'category', + }; + if (!_.has(collectionFields, collection)) { + throw new Error(`invalid collection '${collection}'`); + } + + const collectionField = collectionFields[collection]; + const itemType = { + podcast: 'podcastEpisode', + category: 'meditation', + }[collectionField]; + + const path = 'entries'; + const params = { + content_type: itemType, + [`fields.${collectionField}.sys.id`]: collectionId, + }; + const { campaign_url: campaignUrl } = await getCreds('patreon'); + const patreon = { + token: _.get(req.query, 'patreonToken'), + campaignUrl, + }; + const access = await canAccessFeed(collection, collectionId, patreon); + if (!access) { + res.status(401).send('feed access denied'); + return; + } + res.status(200).json({ success: true }); + return; + + const data = await contentfulGet(path, params, patreon.token); + // TODO: generate feed + }, + )); + app.use( // eslint-disable-next-line async (error, req, res, next) => { diff --git a/package-lock.json b/package-lock.json index 9c28a29..bf6e9a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -244,11 +244,11 @@ }, "dependencies": { "async": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", - "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", "requires": { - "lodash": "^4.17.10" + "lodash": "^4.17.11" } } } @@ -446,9 +446,9 @@ } }, "bluebird": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", - "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==" + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.4.tgz", + "integrity": "sha512-FG+nFEZChJrbQ9tIccIfZJBz3J7mLrAhxakAbnrJWn8d7aKOC+LWifa0G+p4ZqKp4y13T7juYvdhq9NzKdsrjw==" }, "body-parser": { "version": "1.18.3", @@ -612,6 +612,11 @@ "unset-value": "^1.0.0" } }, + "cachedir": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.2.0.tgz", + "integrity": "sha512-VvxA0xhNqIIfg0V9AmJkDg91DaJwryutH5rVEZAhcNi4iJFj9f+QxmAjgK1LT9I8OgToX27fypX6/MeCXVbBjQ==" + }, "callsites": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.0.0.tgz", @@ -844,9 +849,9 @@ } }, "component-emitter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", - "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" }, "compress-commons": { "version": "1.2.2", @@ -2313,9 +2318,9 @@ } }, "ieee754": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.12.tgz", - "integrity": "sha512-GguP+DRY+pJ3soyIiGPTvdiVXjZ+DbXOxGpXn3eMvNW4x4irjqXm4wHKscC+TfxSJ0yw/S1F24tqdMNsMZTiLA==" + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "ienoopen": { "version": "1.0.0", @@ -2328,6 +2333,11 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" + }, "import-fresh": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.0.0.tgz", @@ -2660,9 +2670,9 @@ "dev": true }, "js-yaml": { - "version": "3.12.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.1.tgz", - "integrity": "sha512-um46hB9wNOKlwkHgiuyEVAybXBjwFUV0Z/RaHJblRd9DXltue9FTYvzCr9ErQrK9Adz5MU4gHWVaNUfdmrC8qA==", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -2688,9 +2698,9 @@ }, "dependencies": { "commander": { - "version": "2.19.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", - "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==" + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==" } } }, @@ -2724,6 +2734,17 @@ "graceful-fs": "^4.1.6" } }, + "jszip": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.2.1.tgz", + "integrity": "sha512-iCMBbo4eE5rb1VCpm5qXOAaUiRKRUKiItn8ah2YQQx9qymmSAY98eyQfioChEYcVQLh0zxJ3wS4A0mh90AVPvw==", + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, "jwt-decode": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-2.2.0.tgz", @@ -2776,6 +2797,14 @@ "type-check": "~0.3.2" } }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "requires": { + "immediate": "~3.0.5" + } + }, "load-json-file": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", @@ -3398,6 +3427,11 @@ "semver": "^5.1.0" } }, + "pako": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.10.tgz", + "integrity": "sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==" + }, "parent-module": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.0.tgz", @@ -3447,9 +3481,9 @@ "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" }, "path-loader": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/path-loader/-/path-loader-1.0.9.tgz", - "integrity": "sha512-pD37gArtr+/72Tst9oJoDB9k7gB9A09Efj7yyBi5HDUqaxqULXBWW8Rnw2TfNF+3sN7QZv0ZNdW1Qx2pFGW5Jg==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/path-loader/-/path-loader-1.0.10.tgz", + "integrity": "sha512-CMP0v6S6z8PHeJ6NFVyVJm6WyJjIwFvyz2b0n2/4bKdS/0uZa/9sKUlYZzubrn3zuDRU0zIuEDX9DZYQ2ZI8TA==", "requires": { "native-promise-only": "^0.8.1", "superagent": "^3.8.3" @@ -3704,9 +3738,9 @@ "dev": true }, "registry-auth-token": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.2.tgz", - "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", "requires": { "rc": "^1.1.6", "safe-buffer": "^5.0.1" @@ -3782,6 +3816,30 @@ "glob": "^7.1.3" } }, + "rss": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/rss/-/rss-1.2.2.tgz", + "integrity": "sha1-UKFpiHYTgTOnT5oF0r3I240nqSE=", + "requires": { + "mime-types": "2.1.13", + "xml": "1.0.1" + }, + "dependencies": { + "mime-db": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.25.0.tgz", + "integrity": "sha1-wY29fHOl2/b0SgJNwNFloeexw5I=" + }, + "mime-types": { + "version": "2.1.13", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.13.tgz", + "integrity": "sha1-4HqqnGxrmnyjASxpADrSWjnpKog=", + "requires": { + "mime-db": "~1.25.0" + } + } + } + }, "rss-parser": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.7.0.tgz", @@ -3914,14 +3972,15 @@ } }, "serverless": { - "version": "1.36.3", - "resolved": "https://registry.npmjs.org/serverless/-/serverless-1.36.3.tgz", - "integrity": "sha512-tFiePa29gHrbG3EbmegVeScUsJrDP+sB93nGPNW/R6DGccqxcI+84zZ2ICSoIF/DwH0YpmT4kffrqEzpadlKCw==", + "version": "1.42.1", + "resolved": "https://registry.npmjs.org/serverless/-/serverless-1.42.1.tgz", + "integrity": "sha512-h/ZzXz03EbxYIkTtZ4+MciWJjejz2xFp7m6FIi/GXHTmDdj3tLucUxWMExieCMpCzGD9qWP2wXb3T0wLvoF0Fw==", "requires": { "archiver": "^1.1.0", "async": "^1.5.2", - "aws-sdk": "^2.373.0", + "aws-sdk": "^2.430.0", "bluebird": "^3.5.0", + "cachedir": "^2.2.0", "chalk": "^2.0.0", "ci-info": "^1.1.1", "download": "^5.0.2", @@ -3933,12 +3992,14 @@ "graceful-fs": "^4.1.11", "https-proxy-agent": "^2.2.1", "is-docker": "^1.1.0", - "js-yaml": "^3.6.1", + "js-yaml": "^3.13.0", "json-cycle": "^1.3.0", "json-refs": "^2.1.5", + "jszip": "^3.2.1", "jwt-decode": "^2.2.0", "lodash": "^4.13.1", "minimist": "^1.2.0", + "mkdirp": "^0.5.1", "moment": "^2.13.0", "nanomatch": "^1.2.13", "node-fetch": "^1.6.0", @@ -3947,7 +4008,7 @@ "raven": "^1.2.1", "rc": "^1.1.6", "replaceall": "^0.1.6", - "semver": "^5.0.3", + "semver": "^5.7.0", "semver-regex": "^1.0.0", "tabtab": "^2.2.2", "untildify": "^3.0.3", @@ -3955,6 +4016,60 @@ "uuid": "^2.0.2", "write-file-atomic": "^2.1.0", "yaml-ast-parser": "0.0.34" + }, + "dependencies": { + "aws-sdk": { + "version": "2.452.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.452.0.tgz", + "integrity": "sha512-l6J2NmUg12xpDKG9YdlPje5+Z+nNvqyWMA85ookzPqwx8RcD28D3vg4K1aGi27/oo/3NsngR3XfKUBPSqUpUMA==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.8", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "ieee754": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" + } } }, "serverless-http": { @@ -3967,6 +4082,11 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, + "set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" + }, "set-value": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", @@ -4581,29 +4701,12 @@ "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, "unbzip2-stream": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.2.tgz", - "integrity": "sha512-l71qM60cLs5GjR4uJsACOADWuttjtGNLcQdOj6FxSkxhovPiX2+pm+mERrYjkYqAx9EZoQh3LD61oSYx0tycww==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", + "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", "requires": { - "buffer": "^3.0.1", - "through": "^2.3.6" - }, - "dependencies": { - "base64-js": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-0.0.8.tgz", - "integrity": "sha1-EQHpVE9KdrG8OybUUsqW16NeeXg=" - }, - "buffer": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-3.6.0.tgz", - "integrity": "sha1-pyyTb3e5a/UvX357RnGAYoVR3vs=", - "requires": { - "base64-js": "0.0.8", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - } + "buffer": "^5.2.1", + "through": "^2.3.8" } }, "union-value": { @@ -4905,6 +5008,11 @@ "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=" }, + "xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=" + }, "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", diff --git a/package.json b/package.json index dd2c148..2847601 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,9 @@ "patreon": "^0.4.1", "progress": "^2.0.3", "qs": "^6.6.0", + "rss": "^1.2.2", "rss-parser": "^3.7.0", - "serverless": "^1.36.3", + "serverless": "^1.42.1", "serverless-http": "^1.9.0", "yargs": "^13.2.2" }, diff --git a/serverless.yml b/serverless.yml index 96f1416..46fe7f0 100644 --- a/serverless.yml +++ b/serverless.yml @@ -5,6 +5,8 @@ provider: runtime: nodejs8.10 stage: dev region: us-east-1 + tracing: + lambda: true environment: SLS_STAGE: ${self:custom.stage} iamRoleStatements: @@ -32,6 +34,7 @@ functions: - http: GET /patreon/authorize - http: POST /patreon/validate - http: GET /contentful/{proxy+} + - http: GET /rss/{proxy+} sync: handler: src/sync.handler timeout: 300 From 0ac2ae4c2f8fd6064b1b3066b6f6f0a724639cca Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 14 May 2019 21:56:34 -0400 Subject: [PATCH 11/52] Make it a real podcast (more fields) --- index.js | 199 ++++++++++++++++++++++++++++++++++++---------- package-lock.json | 5 ++ package.json | 2 + 3 files changed, 164 insertions(+), 42 deletions(-) diff --git a/index.js b/index.js index 11dddcc..3470131 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,8 @@ const morgan = require('morgan'); const _ = require('lodash'); const Sentry = require('@sentry/node'); const { patreon: patreonAPI } = require('patreon'); +const RSS = require('rss'); +const striptags = require('striptags'); const { getCreds } = require('./src/creds'); @@ -31,7 +33,7 @@ function canAccessPatronMedia(pledge) { } function canAccess(pledge, item, podcasts) { - const contentType = item.sys.contentType.sys.id; + const contentType = _.get(item, 'sys.contentType.sys.id'); if (contentType === 'podcastEpisode') { const podcast = podcasts[item.fields.podcast.sys.id]; return canAccessPodcast(pledge, podcast); @@ -74,37 +76,44 @@ async function getPledge(patreon) { ); } +function filterEntry(entry, pledge, podcasts) { + if (canAccess(pledge, entry, podcasts)) { + return _.set(entry, 'fields.patronsOnly', false); + } + + const filteredEntry = entry.fields.isFreePreview + ? entry + : _.omit(entry, ['fields.media', 'fields.mediaUrl']); + + // `patronsOnly` tells the app that the user can't access this entry + // because they're not a patron or haven't pledged enough. + // `isFreePreview` tells the app to put a little "Free Preview" + // label on entries that wouldn't have been ordinarily accessible but are + // given as preview media. + return _.set(filteredEntry, 'fields.patronsOnly', true); +} + async function filterData(contentfulData, patreon) { - const pledge = getPledge(patreon); + const pledge = await getPledge(patreon); + + if (!_.has(contentfulData, 'items')) { + // this is not an entry set; it's a single entry + return filterEntry(contentfulData, pledge, {}); + } // podcasts are included; pull them out by ID first + const includedEntries = _.get(contentfulData, 'includes.Entry', []); const podcasts = _.fromPairs( - contentfulData.includes.Entry.filter( + includedEntries.filter( entry => entry.sys.contentType.sys.id === 'podcast', ).map(podcast => ([podcast.sys.id, podcast])), ); - return { + const d = { ...contentfulData, - items: contentfulData.items.map( - (item) => { - if (canAccess(pledge, item, podcasts)) { - return _.set(item, 'fields.patronsOnly', false); - } - - const filteredItem = item.fields.isFreePreview - ? item - : _.omit(item, ['fields.media', 'fields.mediaUrl']); - - // `patronsOnly` tells the app that the user can't access this item - // because they're not a patron or haven't pledged enough. - // `isFreePreview` tells the app to put a little "Free Preview" - // label on items that wouldn't have been ordinarily accessible but are - // given as preview media. - return _.set(filteredItem, 'fields.patronsOnly', true); - }, - ), + items: contentfulData.items.map(item => filterEntry(item, pledge, podcasts)), }; + return d; } function wrapAsync(fn) { @@ -209,9 +218,8 @@ async function init() { token: patreonToken, campaignUrl, }; - const { status } = contentfulRes; - const { data } = contentfulRes; - Sentry.addBreadcrumb({ message: 'Got contentful response', data }); + const { status, data } = contentfulRes; + Sentry.addBreadcrumb({ message: 'Got contentful response', data: { json: data } }); if (status >= 400) { const e = new Error(); @@ -233,7 +241,7 @@ async function init() { + 'Please re-connect Patreon and try again.' ), }; - throw e.error; + throw e; } } @@ -241,7 +249,7 @@ async function init() { async (req, res) => { Sentry.addBreadcrumb({ message: 'API request', data: req.path }); - const path = req.params[1]; + const path = req.params[1]; // second wildcard match const params = req.query; const patreonToken = _.get(req.headers, 'x-theliturgists-patreon-token'); @@ -255,16 +263,47 @@ async function init() { }, )); - async function canAccessFeed(collection, collectionId, patreon) { + async function canAccessFeed(collectionObj, patreon) { const pledge = await getPledge(patreon); - if (collection === 'podcast') { - const path = `entries/${collectionId}`; - const podcast = await contentfulGet(path, {}, patreon.token, false); - return canAccessPodcast(pledge, podcast); + if (collectionObj.sys.contentType.sys.id === 'podcast') { + return canAccessPodcast(pledge, collectionObj); } return canAccessPatronMedia(pledge); } + function getImageUrl(collectionObj) { + let url = _.get(collectionObj, 'fields.image.file.url'); + if (!url) { + url = _.get(collectionObj, 'fields.imageUrl'); + } + return url; + } + + function getMediaUrl(entry) { + let url = _.get(entry, 'fields.media.file.url'); + if (!url) { + url = _.get(entry, 'fields.mediaUrl'); + } + return url; + } + + function imageElementIfDefined(entry) { + const href = getImageUrl(entry); + if (!href) { + return []; + } + + return [ + { 'itunes:image': { _attr: { href } } }, + ]; + } + + function getFullRequestUrl(req) { + const { hostname, path } = req; + const prefix = /execute-api/.test(hostname) ? `/${stage}` : ''; + return `https://${hostname}${prefix}${path}`; + } + app.get('*/rss/:collection/:collectionId', wrapAsync( async (req, res) => { const { collection, collectionId } = req.params; @@ -282,26 +321,102 @@ async function init() { category: 'meditation', }[collectionField]; - const path = 'entries'; - const params = { - content_type: itemType, - [`fields.${collectionField}.sys.id`]: collectionId, - }; const { campaign_url: campaignUrl } = await getCreds('patreon'); const patreon = { token: _.get(req.query, 'patreonToken'), campaignUrl, }; - const access = await canAccessFeed(collection, collectionId, patreon); + const collectionObj = await contentfulGet( + `entries/${collectionId}`, + {}, + patreon.token, + false, + ); + const access = await canAccessFeed(collectionObj, patreon); if (!access) { res.status(401).send('feed access denied'); return; } - res.status(200).json({ success: true }); - return; + if (collectionObj.fields.image) { + const assetId = collectionObj.fields.image.sys.id; + const imageAsset = await contentfulGet(`assets/${assetId}`, {}); + imageAsset.fields.file.url = `https:${imageAsset.fields.file.url}`; + collectionObj.fields.image = _.pick(imageAsset.fields, ['file', 'title']); + } + + const params = { + content_type: itemType, + [`fields.${collectionField}.sys.id`]: collectionId, + order: '-fields.publishedAt', + }; + const data = await contentfulGet('entries', params, patreon.token); + const category = 'Religion & Spirituality'; + const author = 'The Liturgists Network'; + const feed = new RSS({ + title: collectionObj.fields.title, + description: collectionObj.fields.description, + feed_url: getFullRequestUrl(req), + site_url: 'https://theliturgists.com', + image_url: getImageUrl(collectionObj), + language: 'en-US', + categories: [category], + pubDate: _.get(data.items, [0, 'fields', 'publishedAt'], null), + custom_namespaces: { + itunes: 'http://www.itunes.com/dtds/podcast-1.0.dtd', + googleplay: 'http://www.google.com/schemas/play-podcasts/1.0', + }, + custom_elements: [ + { 'itunes:block': 'yes' }, + { 'googleplay:block': 'yes' }, + { 'itunes:summary': striptags(collectionObj.fields.description) }, + { 'itunes:author': author }, + { + 'itunes:owner': [ + { 'itunes:name': author }, + { 'itunes:email': 'app@theliturgists.com' }, + ], + }, + { + 'itunes:image': { + _attr: { href: getImageUrl(collectionObj) }, + }, + }, + { + 'itunes:category': { + _attr: { text: category }, + }, + }, + ], + }); - const data = await contentfulGet(path, params, patreon.token); - // TODO: generate feed + // TODO: handle contentful pagination + data.items.forEach((entry) => { + feed.item({ + guid: entry.sys.id, + title: entry.fields.title, + description: entry.fields.description, + date: entry.fields.publishedAt, + enclosure: { + url: getMediaUrl(entry), + type: 'audio/mpeg', + }, + image_url: getImageUrl(entry), + custom_elements: [ + ...imageElementIfDefined(entry), + { 'itunes:duration': entry.fields.duration }, + { 'itunes:summary': striptags(entry.fields.description) }, + { 'itunes:subtitle': striptags(entry.fields.description) }, + { + 'content:encoded': { + _cdata: entry.fields.description, + }, + }, + ], + }); + }); + const xml = feed.xml({ indent: true }); + res.set('Content-type', 'application/rss+xml'); + res.status(200).send(xml); }, )); diff --git a/package-lock.json b/package-lock.json index bf6e9a7..e67db70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4476,6 +4476,11 @@ "escape-string-regexp": "^1.0.2" } }, + "striptags": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/striptags/-/striptags-3.1.1.tgz", + "integrity": "sha1-yMPn/db7S7OjKjt1LltePjgJPr0=" + }, "superagent": { "version": "3.8.3", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.3.tgz", diff --git a/package.json b/package.json index 2847601..3d25d5f 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "contentful-management": "^5.7.0", "express": "^4.16.4", "helmet": "^3.15.1", + "json-cycle": "^1.3.0", "lodash": "^4.17.11", "morgan": "^1.9.1", "patreon": "^0.4.1", @@ -27,6 +28,7 @@ "rss-parser": "^3.7.0", "serverless": "^1.42.1", "serverless-http": "^1.9.0", + "striptags": "^3.1.1", "yargs": "^13.2.2" }, "devDependencies": { From 35b37a5d32c3c19a9a19be19514df0aeccbe7ef7 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 15 May 2019 08:14:16 -0400 Subject: [PATCH 12/52] Redirect to public feed URL when available --- index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.js b/index.js index 3470131..1880b85 100644 --- a/index.js +++ b/index.js @@ -337,6 +337,11 @@ async function init() { res.status(401).send('feed access denied'); return; } + if (collectionObj.fields.feedUrl) { + res.redirect(collectionObj.fields.feedUrl); + return; + } + if (collectionObj.fields.image) { const assetId = collectionObj.fields.image.sys.id; const imageAsset = await contentfulGet(`assets/${assetId}`, {}); From 8770891470b24503fefda63271cda0d679f7466f Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 21 May 2019 08:27:54 -0400 Subject: [PATCH 13/52] Store Patreon token and return liturgists token --- .eslintrc.js | 1 + aws/deploy_policy.json | 8 + index.js | 437 ++++++++++++++++++++++++++--------------- package-lock.json | 291 ++++++++++++++++++++++++++- package.json | 5 + serverless.yml | 61 +++++- src/TokenMapping.js | 30 +++ src/creds.js | 3 + 8 files changed, 678 insertions(+), 158 deletions(-) create mode 100644 src/TokenMapping.js diff --git a/.eslintrc.js b/.eslintrc.js index 1bd1001..9a79752 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -3,5 +3,6 @@ module.exports = { "rules": { "no-underscore-dangle": [ "error", { "allowAfterThis": true } ], "import/prefer-default-export": [ "off" ], + "prefer-destructuring": [ "off " ], } }; diff --git a/aws/deploy_policy.json b/aws/deploy_policy.json index ea0cf1f..2e77b06 100644 --- a/aws/deploy_policy.json +++ b/aws/deploy_policy.json @@ -67,6 +67,7 @@ "cloudformation:Describe*", "cloudformation:List*", "cloudformation:Get*", + "cloudformation:SetStackPolicy", "cloudformation:PreviewStackUpdate", "cloudformation:CreateStack", "cloudformation:UpdateStack", @@ -197,6 +198,13 @@ ], "Resource": "*", "Effect": "Allow" + }, + { + "Action": [ + "dynamodb:*Table*" + ], + "Resource": "*", + "Effect": "Allow" } ] } diff --git a/index.js b/index.js index 1880b85..4b9f876 100644 --- a/index.js +++ b/index.js @@ -10,8 +10,12 @@ const Sentry = require('@sentry/node'); const { patreon: patreonAPI } = require('patreon'); const RSS = require('rss'); const striptags = require('striptags'); +const uuidv4 = require('uuid/v4'); +const jwt = require('jsonwebtoken'); +const moment = require('moment'); const { getCreds } = require('./src/creds'); +const TokenMapping = require('./src/TokenMapping'); const stage = process.env.SLS_STAGE; @@ -29,7 +33,8 @@ function canAccessPatronMedia(pledge) { // Patrons with the 'Master Meditations' reward tier // get access to both Meditations and Liturgies // in addition to patrons-only podcasts. - return pledge && /Meditations/i.test(pledge.reward.title); + const title = _.get(pledge, 'reward.title'); + return title && /Meditations/i.test(title); } function canAccess(pledge, item, podcasts) { @@ -44,12 +49,7 @@ function canAccess(pledge, item, podcasts) { return true; } -async function getPledge(patreon) { - const { token, campaignUrl } = patreon; - if (!token || !campaignUrl) { - return null; - } - +async function getPatreonUser(token) { const client = patreonAPI(token); let resp; try { @@ -68,7 +68,16 @@ async function getPledge(patreon) { } const { store, rawJson } = resp; - const user = store.find('user', rawJson.data.id); + return store.find('user', rawJson.data.id); +} + +async function getPledge(patreon) { + const { token, campaignUrl } = patreon; + if (!token || !campaignUrl) { + return null; + } + + const user = await getPatreonUser(token); return _.find( user.pledges, @@ -77,6 +86,7 @@ async function getPledge(patreon) { } function filterEntry(entry, pledge, podcasts) { + console.log(entry, podcasts); if (canAccess(pledge, entry, podcasts)) { return _.set(entry, 'fields.patronsOnly', false); } @@ -93,9 +103,7 @@ function filterEntry(entry, pledge, podcasts) { return _.set(filteredEntry, 'fields.patronsOnly', true); } -async function filterData(contentfulData, patreon) { - const pledge = await getPledge(patreon); - +async function filterData(contentfulData, pledge) { if (!_.has(contentfulData, 'items')) { // this is not an entry set; it's a single entry return filterEntry(contentfulData, pledge, {}); @@ -126,9 +134,64 @@ function wrapAsync(fn) { ); } -const patreonBaseUrl = 'https://www.patreon.com/'; +const patreonBaseUrl = 'https://www.patreon.com'; const patreonAuthUrl = `${patreonBaseUrl}/oauth2/authorize`; -const patreonTokenUrl = `${patreonBaseUrl}/api/oauth2/token`; +const patreonApiUrl = `${patreonBaseUrl}/api/oauth2`; +const patreonTokenUrl = `${patreonApiUrl}/token`; + +/** + * Return a route handler that maps the liturgists token (if present) + * to the stored Patreon token, retrieves the patron's pledge (if any), + * attaches it to the request object as req.pledge, and passes control + * to the next function. + * + * Can be used in any route that supplies the `x-theliturgists-token` + * header with the JWT that was returned from the shimmed Patreon + * OAuth flow. + */ +function handleLiturgistsToken() { + return wrapAsync( + async (req, res, next) => { + const token = _.get( + req.headers, + 'x-theliturgists-token', + _.get(req.query, 'token') + ); + + if (token) { + const { secret } = await getCreds('jwt'); + try { + const { userId } = jwt.verify(token, secret); + const resp = await TokenMapping + .query(userId) + .exec() + .promise(); + const [{ Items: items }] = resp; + if (!items || items.length === 0) { + throw new Error(`no token mapping found for userId ${userId}`); + } + console.log('token mapping:', items[0].attrs); + const { patreonToken } = items[0].attrs; + if (!patreonToken) { + throw new Error(`no patreon token found for userId ${userId}`); + } + const { campaign_url: campaignUrl } = await getCreds('patreon'); + + // Assign token mapping and pledge to request object for later use + req.tokenMapping = items[0].attrs; + req.pledge = await getPledge({ token: patreonToken, campaignUrl }); + console.log('pledge:', req.pledge); + } catch (err) { + console.error(err); + res.status(401).json({ error: 'invalid token' }); + return; + } + } + + next(); + }, + ); +} async function init() { const sentry = await getCreds('sentry'); @@ -170,16 +233,87 @@ async function init() { bodyParser.urlencoded({ extended: true }), wrapAsync( async (req, res) => { - const patreon = await getCreds('patreon'); + // eslint-disable-next-line camelcase + const { client_id, client_secret } = await getCreds('patreon'); const url = patreonTokenUrl; const obj = { ...req.body, - ..._.pick(patreon, ['client_id', 'client_secret']), + client_id, + client_secret, }; const body = qs.stringify(obj); - const patreonRes = await axios.post(url, body, { validateStatus: null }); + let patreonRes; + try { + patreonRes = await axios.post(url, body); + } catch (err) { + res.status(err.response.status).json(err.response.data); + return; + } + + const { + access_token: patreonToken, + refresh_token: refreshToken, + expires_in: expiresIn, + } = patreonRes.data; + + const expiresAt = moment().add(expiresIn, 'seconds').toISOString(); + + const patreonUser = await getPatreonUser(patreonToken); + const patreonUserId = patreonUser.id; + + const resp = await TokenMapping + .query(patreonUserId) + .usingIndex('patreonUserIdIndex') + .exec() + .promise(); + + console.log('TokenMapping resp:', resp); + + const [{ Items: items }] = resp; + console.log(`Destroying ${items.length} existing patreon records`); + await Promise.all(items.map(item => item.destroy())); + + const userId = uuidv4(); + await TokenMapping.create({ + userId, + patreonUserId, + patreonToken, + refreshToken, + expiresAt, + }); + + const { secret } = await getCreds('jwt'); + const token = jwt.sign({ userId }, secret); + const data = { + liturgistsToken: token, + }; + + res.status(200).json(data); + }, + ), + ); + + app.get( + '*/patreon/api/*', + handleLiturgistsToken(), + wrapAsync( + async (req, res) => { + const path = req.params[1]; // second wildcard match + const url = `${patreonApiUrl}/api/${path}`; + console.log('tokenMapping:', req.tokenMapping); + const token = req.tokenMapping.patreonToken; + const patreonRes = await axios.get( + url, + { + validateStatus: null, + headers: { + authorization: `Bearer ${token}`, + }, + } + ); + console.log('patreon response', patreonRes); res.status(patreonRes.status).json(patreonRes.data); }, ), @@ -195,10 +329,9 @@ async function init() { * @throws {Error} on contentful API error * error object has 'status' and 'json' fields */ - async function contentfulGet(path, params, patreonToken, filter = true) { + async function contentfulGet(path, params, pledge, filter = true) { const contentful = await getCreds('contentful'); const { space, environment } = contentful; - const { campaign_url: campaignUrl } = await getCreds('patreon'); const fullPath = `/spaces/${space}/environments/${environment}/${path}`; const host = 'https://cdn.contentful.com'; @@ -214,10 +347,6 @@ async function init() { authorization: `Bearer ${contentful.accessToken}`, }, }); - const patreon = { - token: patreonToken, - campaignUrl, - }; const { status, data } = contentfulRes; Sentry.addBreadcrumb({ message: 'Got contentful response', data: { json: data } }); @@ -233,7 +362,7 @@ async function init() { } try { - return await filterData(data, patreon); + return await filterData(data, pledge); } catch (e) { e.json = { error: ( @@ -245,26 +374,28 @@ async function init() { } } - app.get('*/contentful/spaces/:space/environments/:env/*', wrapAsync( - async (req, res) => { - Sentry.addBreadcrumb({ message: 'API request', data: req.path }); - - const path = req.params[1]; // second wildcard match - const params = req.query; - const patreonToken = _.get(req.headers, 'x-theliturgists-patreon-token'); - - try { - const data = await contentfulGet(path, params, patreonToken); - res.status(200).json(data); - } catch (e) { - console.log(e); - res.status(e.status).json(e.json); - } - }, - )); + app.get( + '*/contentful/spaces/:space/environments/:env/*', + handleLiturgistsToken(), + wrapAsync( + async (req, res) => { + Sentry.addBreadcrumb({ message: 'API request', data: req.path }); + + const path = req.params[1]; // second wildcard match + const params = req.query; + + try { + const data = await contentfulGet(path, params, req.pledge); + res.status(200).json(data); + } catch (e) { + console.error(e); + res.status(e.status).json(e.json); + } + }, + ), + ); - async function canAccessFeed(collectionObj, patreon) { - const pledge = await getPledge(patreon); + async function canAccessFeed(collectionObj, pledge) { if (collectionObj.sys.contentType.sys.id === 'podcast') { return canAccessPodcast(pledge, collectionObj); } @@ -304,126 +435,124 @@ async function init() { return `https://${hostname}${prefix}${path}`; } - app.get('*/rss/:collection/:collectionId', wrapAsync( - async (req, res) => { - const { collection, collectionId } = req.params; - const collectionFields = { - podcast: 'podcast', - 'meditation-category': 'category', - }; - if (!_.has(collectionFields, collection)) { - throw new Error(`invalid collection '${collection}'`); - } - - const collectionField = collectionFields[collection]; - const itemType = { - podcast: 'podcastEpisode', - category: 'meditation', - }[collectionField]; - - const { campaign_url: campaignUrl } = await getCreds('patreon'); - const patreon = { - token: _.get(req.query, 'patreonToken'), - campaignUrl, - }; - const collectionObj = await contentfulGet( - `entries/${collectionId}`, - {}, - patreon.token, - false, - ); - const access = await canAccessFeed(collectionObj, patreon); - if (!access) { - res.status(401).send('feed access denied'); - return; - } - if (collectionObj.fields.feedUrl) { - res.redirect(collectionObj.fields.feedUrl); - return; - } - - if (collectionObj.fields.image) { - const assetId = collectionObj.fields.image.sys.id; - const imageAsset = await contentfulGet(`assets/${assetId}`, {}); - imageAsset.fields.file.url = `https:${imageAsset.fields.file.url}`; - collectionObj.fields.image = _.pick(imageAsset.fields, ['file', 'title']); - } - - const params = { - content_type: itemType, - [`fields.${collectionField}.sys.id`]: collectionId, - order: '-fields.publishedAt', - }; - const data = await contentfulGet('entries', params, patreon.token); - const category = 'Religion & Spirituality'; - const author = 'The Liturgists Network'; - const feed = new RSS({ - title: collectionObj.fields.title, - description: collectionObj.fields.description, - feed_url: getFullRequestUrl(req), - site_url: 'https://theliturgists.com', - image_url: getImageUrl(collectionObj), - language: 'en-US', - categories: [category], - pubDate: _.get(data.items, [0, 'fields', 'publishedAt'], null), - custom_namespaces: { - itunes: 'http://www.itunes.com/dtds/podcast-1.0.dtd', - googleplay: 'http://www.google.com/schemas/play-podcasts/1.0', - }, - custom_elements: [ - { 'itunes:block': 'yes' }, - { 'googleplay:block': 'yes' }, - { 'itunes:summary': striptags(collectionObj.fields.description) }, - { 'itunes:author': author }, - { - 'itunes:owner': [ - { 'itunes:name': author }, - { 'itunes:email': 'app@theliturgists.com' }, - ], + app.get( + '*/rss/:collection/:collectionId', + handleLiturgistsToken(), + wrapAsync( + async (req, res) => { + const { collection, collectionId } = req.params; + const collectionFields = { + podcast: 'podcast', + 'meditation-category': 'category', + }; + if (!_.has(collectionFields, collection)) { + throw new Error(`invalid collection '${collection}'`); + } + + const collectionField = collectionFields[collection]; + const itemType = { + podcast: 'podcastEpisode', + category: 'meditation', + }[collectionField]; + + const collectionObj = await contentfulGet( + `entries/${collectionId}`, + {}, + req.pledge, + false, + ); + const access = await canAccessFeed(collectionObj, req.pledge); + if (!access) { + res.status(401).send('feed access denied'); + return; + } + if (collectionObj.fields.feedUrl) { + res.redirect(collectionObj.fields.feedUrl); + return; + } + + if (collectionObj.fields.image) { + const assetId = collectionObj.fields.image.sys.id; + const imageAsset = await contentfulGet(`assets/${assetId}`, {}); + imageAsset.fields.file.url = `https:${imageAsset.fields.file.url}`; + collectionObj.fields.image = _.pick(imageAsset.fields, ['file', 'title']); + } + + const params = { + content_type: itemType, + [`fields.${collectionField}.sys.id`]: collectionId, + order: '-fields.publishedAt', + }; + const data = await contentfulGet('entries', params, req.pledge); + const category = 'Religion & Spirituality'; + const author = 'The Liturgists Network'; + const feed = new RSS({ + title: collectionObj.fields.title, + description: collectionObj.fields.description, + feed_url: getFullRequestUrl(req), + site_url: 'https://theliturgists.com', + image_url: getImageUrl(collectionObj), + language: 'en-US', + categories: [category], + pubDate: _.get(data.items, [0, 'fields', 'publishedAt'], null), + custom_namespaces: { + itunes: 'http://www.itunes.com/dtds/podcast-1.0.dtd', + googleplay: 'http://www.google.com/schemas/play-podcasts/1.0', }, - { - 'itunes:image': { - _attr: { href: getImageUrl(collectionObj) }, + custom_elements: [ + { 'itunes:block': 'yes' }, + { 'googleplay:block': 'yes' }, + { 'itunes:summary': striptags(collectionObj.fields.description) }, + { 'itunes:author': author }, + { + 'itunes:owner': [ + { 'itunes:name': author }, + { 'itunes:email': 'app@theliturgists.com' }, + ], }, - }, - { - 'itunes:category': { - _attr: { text: category }, + { + 'itunes:image': { + _attr: { href: getImageUrl(collectionObj) }, + }, }, - }, - ], - }); - - // TODO: handle contentful pagination - data.items.forEach((entry) => { - feed.item({ - guid: entry.sys.id, - title: entry.fields.title, - description: entry.fields.description, - date: entry.fields.publishedAt, - enclosure: { - url: getMediaUrl(entry), - type: 'audio/mpeg', - }, - image_url: getImageUrl(entry), - custom_elements: [ - ...imageElementIfDefined(entry), - { 'itunes:duration': entry.fields.duration }, - { 'itunes:summary': striptags(entry.fields.description) }, - { 'itunes:subtitle': striptags(entry.fields.description) }, { - 'content:encoded': { - _cdata: entry.fields.description, + 'itunes:category': { + _attr: { text: category }, }, }, ], }); - }); - const xml = feed.xml({ indent: true }); - res.set('Content-type', 'application/rss+xml'); - res.status(200).send(xml); - }, - )); + + // TODO: handle contentful pagination + data.items.forEach((entry) => { + feed.item({ + guid: entry.sys.id, + title: entry.fields.title, + description: entry.fields.description, + date: entry.fields.publishedAt, + enclosure: { + url: getMediaUrl(entry), + type: 'audio/mpeg', + }, + image_url: getImageUrl(entry), + custom_elements: [ + ...imageElementIfDefined(entry), + { 'itunes:duration': entry.fields.duration }, + { 'itunes:summary': striptags(entry.fields.description) }, + { 'itunes:subtitle': striptags(entry.fields.description) }, + { + 'content:encoded': { + _cdata: entry.fields.description, + }, + }, + ], + }); + }); + const xml = feed.xml({ indent: true }); + res.set('Content-type', 'application/rss+xml'); + res.status(200).send(xml); + }, + )); app.use( // eslint-disable-next-line diff --git a/package-lock.json b/package-lock.json index e67db70..08f0119 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,34 @@ "is-buffer": "^1.1.5" } }, + "@hapi/address": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.0.0.tgz", + "integrity": "sha512-mV6T0IYqb0xL1UALPFplXYQmR0twnXG0M6jUswpquqT2sD12BOiCiLy3EvMp/Fy7s3DZElC4/aPjEjo2jeZpvw==" + }, + "@hapi/hoek": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-6.2.1.tgz", + "integrity": "sha512-+ryw4GU9pjr1uT6lBuErHJg3NYqzwJTvZ75nKuJijEzpd00Uqi6oiawTGDDf5Hl0zWmI7qHfOtaqB0kpQZJQzA==" + }, + "@hapi/joi": { + "version": "15.0.3", + "resolved": "https://registry.npmjs.org/@hapi/joi/-/joi-15.0.3.tgz", + "integrity": "sha512-z6CesJ2YBwgVCi+ci8SI8zixoj8bGFn/vZb9MBPbSyoxsS2PnWYjHcyTM17VLK6tx64YVK38SDIh10hJypB+ig==", + "requires": { + "@hapi/address": "2.x.x", + "@hapi/hoek": "6.x.x", + "@hapi/topo": "3.x.x" + } + }, + "@hapi/topo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-3.1.0.tgz", + "integrity": "sha512-gZDI/eXOIk8kP2PkUKjWu9RW8GGVd2Hkgjxyr/S7Z+JF+0mr7bAlbw+DkTRxnD580o8Kqxlnba9wvqp5aOHBww==", + "requires": { + "@hapi/hoek": "6.x.x" + } + }, "@sentry/core": { "version": "4.6.4", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-4.6.4.tgz", @@ -227,6 +255,11 @@ "color-convert": "^1.9.0" } }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + }, "archiver": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/archiver/-/archiver-1.3.0.tgz", @@ -575,6 +608,11 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + }, "buffer-fill": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", @@ -591,6 +629,16 @@ "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", "dev": true }, + "bunyan": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz", + "integrity": "sha1-X259RMQ7lS9WsPQTCeOrEjkbTi0=", + "requires": { + "dtrace-provider": "~0.6", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, "bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -1201,11 +1249,73 @@ "pify": "^2.3.0" } }, + "dtrace-provider": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "integrity": "sha1-CweNVReTfYcxAUUtkUZzdVe3XlE=", + "optional": true, + "requires": { + "nan": "^2.0.8" + } + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" }, + "dynamodb": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/dynamodb/-/dynamodb-1.2.0.tgz", + "integrity": "sha512-O28acIHevSLRL7GOzX2oKgPD2tQnzjJsy92L27hjUGUXz+ETgLRTVw5lOujIIpLJBncDNJlx1KN1qfBwp2AmMw==", + "requires": { + "async": "1.5.x", + "aws-sdk": "^2.186.x", + "bunyan": "1.5.x", + "joi": "10.6.x", + "lodash": "4.x.x", + "node-uuid": "1.4.x", + "stream-to-promise": "~2.2.0" + }, + "dependencies": { + "hoek": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", + "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" + }, + "isemail": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/isemail/-/isemail-2.2.1.tgz", + "integrity": "sha1-A1PT2aYpUQgMJiwqoKQrjqjp4qY=" + }, + "joi": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/joi/-/joi-10.6.0.tgz", + "integrity": "sha512-hBF3LcqyAid+9X/pwg+eXjD2QBZI5eXnBFJYaAkH4SK3mp9QSRiiQnDYlmlz5pccMvnLcJRS4whhDOTCkmsAdQ==", + "requires": { + "hoek": "4.x.x", + "isemail": "2.x.x", + "items": "2.x.x", + "topo": "2.x.x" + } + }, + "topo": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/topo/-/topo-2.0.2.tgz", + "integrity": "sha1-zVYVdSU5BXwNwEkaYhw7xvvh0YI=", + "requires": { + "hoek": "4.x.x" + } + } + } + }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -2658,6 +2768,11 @@ "is-object": "^1.0.1" } }, + "items": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/items/-/items-2.1.2.tgz", + "integrity": "sha512-kezcEqgB97BGeZZYtX/MA8AG410ptURstvnz5RAgyFZ8wQFPMxHY8GpTq+/ZHKT3frSlIthUq7EvLt9xn3TvXg==" + }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", @@ -2734,6 +2849,23 @@ "graceful-fs": "^4.1.6" } }, + "jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "requires": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + } + }, "jszip": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.2.1.tgz", @@ -2745,6 +2877,25 @@ "set-immediate-shim": "~1.0.1" } }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "jwt-decode": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-2.2.0.tgz", @@ -2837,6 +2988,41 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" }, + "lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" + }, + "lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" + }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, "lodash.pad": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.5.1.tgz", @@ -3052,6 +3238,47 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.6.tgz", "integrity": "sha1-SJYrGeFp/R38JAs/HnMXYnu8R9s=" }, + "mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "optional": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, + "dependencies": { + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "optional": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "optional": true, + "requires": { + "glob": "^6.0.1" + } + } + } + }, + "nan": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.13.2.tgz", + "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==", + "optional": true + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -3081,6 +3308,12 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "optional": true + }, "negotiator": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", @@ -3105,6 +3338,11 @@ "is-stream": "^1.0.1" } }, + "node-uuid": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz", + "integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc=" + }, "normalize-package-data": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.2.tgz", @@ -3876,6 +4114,12 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, + "safe-json-stringify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", + "optional": true + }, "safe-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", @@ -4069,6 +4313,11 @@ "version": "5.7.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" + }, + "uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" } } }, @@ -4418,6 +4667,42 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" }, + "stream-to-array": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz", + "integrity": "sha1-u/azn19D7DC8cbq8s3VXrOzzQ1M=", + "requires": { + "any-promise": "^1.1.0" + } + }, + "stream-to-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/stream-to-promise/-/stream-to-promise-2.2.0.tgz", + "integrity": "sha1-se2y4cjLESidG1A8CNPyrvUeZQ8=", + "requires": { + "any-promise": "~1.3.0", + "end-of-stream": "~1.1.0", + "stream-to-array": "~2.3.0" + }, + "dependencies": { + "end-of-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.1.0.tgz", + "integrity": "sha1-6TUyWLqpEIll78QcsO+K3i88+wc=", + "requires": { + "once": "~1.3.0" + } + }, + "once": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", + "integrity": "sha1-suJhVXzkwxTsgwTz+oJmPkKXyiA=", + "requires": { + "wrappy": "1" + } + } + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -4885,9 +5170,9 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", - "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, "validate-npm-package-license": { "version": "3.0.4", diff --git a/package.json b/package.json index 3d25d5f..94907ae 100644 --- a/package.json +++ b/package.json @@ -10,16 +10,20 @@ "auth-sls": "assume-role theliturgists-deploy serverless" }, "dependencies": { + "@hapi/joi": "^15.0.3", "@sentry/node": "^4.6.4", "aws-param-store": "^2.1.0", "aws-sdk": "^2.399.0", "axios": "^0.18.0", "body-parser": "^1.18.3", "contentful-management": "^5.7.0", + "dynamodb": "^1.2.0", "express": "^4.16.4", "helmet": "^3.15.1", "json-cycle": "^1.3.0", + "jsonwebtoken": "^8.5.1", "lodash": "^4.17.11", + "moment": "^2.24.0", "morgan": "^1.9.1", "patreon": "^0.4.1", "progress": "^2.0.3", @@ -29,6 +33,7 @@ "serverless": "^1.42.1", "serverless-http": "^1.9.0", "striptags": "^3.1.1", + "uuid": "^3.3.2", "yargs": "^13.2.2" }, "devDependencies": { diff --git a/serverless.yml b/serverless.yml index 46fe7f0..84d9fe4 100644 --- a/serverless.yml +++ b/serverless.yml @@ -1,4 +1,4 @@ -service: theliturgists-backend-${self:custom.namespace} +service: ${self:custom.service}-${self:custom.namespace} provider: name: aws @@ -9,6 +9,7 @@ provider: lambda: true environment: SLS_STAGE: ${self:custom.stage} + DYNAMODB_TABLE_TOKENS: ${self:custom.dynamodbTables.tokens} iamRoleStatements: - Effect: Allow Action: @@ -25,6 +26,36 @@ provider: - Ref: AWS::Region - Ref: AWS::AccountId - "parameter/${self:custom.stage}/*" + - Effect: Allow + Action: + - dynamodb:DescribeTable + - dynamodb:Query + - dynamodb:Scan + - dynamodb:GetItem + - dynamodb:PutItem + - dynamodb:UpdateItem + - dynamodb:DeleteItem + Resource: + - "Fn::GetAtt": [ TokensDynamoDBTable, Arn ] + - Fn::Join: + - "/" + - - "Fn::GetAtt": [ TokensDynamoDBTable, Arn ] + - "index" + - "*" + + stackPolicy: + - Effect: Allow + Action: "Update:*" + Principal: "*" + Resource: "*" + - Effect: "Deny" + Action: ["Update:Replace"] + Principal: "*" + Resource: "LogicalResourceId/TokensDynamoDBTable" + - Effect: "Deny" + Action: ["Update:Delete"] + Principal: "*" + Resource: "LogicalResourceId/TokensDynamoDBTable" functions: app: @@ -33,6 +64,7 @@ functions: events: - http: GET /patreon/authorize - http: POST /patreon/validate + - http: GET /patreon/api/{proxy+} - http: GET /contentful/{proxy+} - http: GET /rss/{proxy+} sync: @@ -44,8 +76,35 @@ functions: enabled: ${self:custom.syncEnabled.${self:custom.stage}} custom: + service: theliturgists-backend namespace: ${env:SLS_NAMESPACE, env:USER} stage: ${opt:stage, self:provider.stage} syncEnabled: dev: false staging: true + dynamodbTables: + tokens: "${self:custom.service}-${self:custom.namespace}-${self:custom.stage}-tokens" + +resources: + Resources: + TokensDynamoDBTable: + # Table to store user id / Patreon id/token mapping + Type: AWS::DynamoDB::Table + Properties: + TableName: ${self:custom.dynamodbTables.tokens} + AttributeDefinitions: + - AttributeName: userId + AttributeType: S + - AttributeName: patreonUserId + AttributeType: S + KeySchema: + - AttributeName: userId + KeyType: HASH + GlobalSecondaryIndexes: + - IndexName: patreonUserIdIndex + KeySchema: + - AttributeName: patreonUserId + KeyType: HASH + Projection: + ProjectionType: ALL + BillingMode: PAY_PER_REQUEST diff --git a/src/TokenMapping.js b/src/TokenMapping.js new file mode 100644 index 0000000..307fcfb --- /dev/null +++ b/src/TokenMapping.js @@ -0,0 +1,30 @@ +const dynamodb = require('dynamodb'); +const Joi = require('@hapi/joi'); + +const TokenMapping = dynamodb.define( + 'TokenMapping', + { + hashKey: 'userId', + timestamps: true, + schema: { + userId: Joi.string(), + patreonUserId: Joi.string(), + patreonToken: Joi.string(), + refreshToken: Joi.string(), + expiresAt: Joi.string(), + }, + indexes: [ + { + hashKey: 'patreonUserId', + type: 'global', + name: 'patreonUserIdIndex', + }, + ], + }, +); + +TokenMapping.config({ + tableName: process.env.DYNAMODB_TABLE_TOKENS, +}); + +module.exports = TokenMapping; diff --git a/src/creds.js b/src/creds.js index 8156999..02bf944 100644 --- a/src/creds.js +++ b/src/creds.js @@ -20,6 +20,9 @@ const credSpecs = { sentry: { dsn: `/${stage}/SENTRY_DSN`, }, + jwt: { + secret: `/${stage}/JWT_SECRET`, + }, }; module.exports.getCreds = async (name) => { From d6991562cdf3d141b2f4bc88bcce5e67e51b362d Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 21 May 2019 23:09:29 -0400 Subject: [PATCH 14/52] Handle 'All Mediations' category --- index.js | 88 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/index.js b/index.js index 4b9f876..900285c 100644 --- a/index.js +++ b/index.js @@ -443,7 +443,7 @@ async function init() { const { collection, collectionId } = req.params; const collectionFields = { podcast: 'podcast', - 'meditation-category': 'category', + 'meditationCategory': 'category', }; if (!_.has(collectionFields, collection)) { throw new Error(`invalid collection '${collection}'`); @@ -455,46 +455,86 @@ async function init() { category: 'meditation', }[collectionField]; - const collectionObj = await contentfulGet( - `entries/${collectionId}`, - {}, - req.pledge, - false, - ); - const access = await canAccessFeed(collectionObj, req.pledge); + let access, collectionObj; + if (collection === 'meditationCategory' && collectionId === 'all') { + // there's no actual meditation category for 'all'; + // just check if the user can access meditations in general. + access = canAccessPatronMedia(req.pledge); + } else { + collectionObj = await contentfulGet( + `entries/${collectionId}`, + {}, + req.pledge, + false, + ); + access = await canAccessFeed(collectionObj, req.pledge); + } if (!access) { res.status(401).send('feed access denied'); return; } - if (collectionObj.fields.feedUrl) { - res.redirect(collectionObj.fields.feedUrl); - return; - } - if (collectionObj.fields.image) { - const assetId = collectionObj.fields.image.sys.id; + let coverImageUrl, title, description; + if (collectionObj) { + ({ title, description } = collectionObj.fields); + + if (collectionObj.fields.feedUrl) { + res.redirect(collectionObj.fields.feedUrl); + return; + } + + if (collectionObj.fields.image) { + const assetId = collectionObj.fields.image.sys.id; + const imageAsset = await contentfulGet(`assets/${assetId}`, {}); + imageAsset.fields.file.url = `https:${imageAsset.fields.file.url}`; + collectionObj.fields.image = _.pick(imageAsset.fields, ['file', 'title']); + } + coverImageUrl = getImageUrl(collectionObj); + } else { + // this is the "All Meditations" pseudo-collection + title = 'All Meditations'; + description = 'All meditations in all categories.'; + + const assetId = '4fw1cG2nsTZ9Upl3jpWDVH'; // ID for the cover image const imageAsset = await contentfulGet(`assets/${assetId}`, {}); imageAsset.fields.file.url = `https:${imageAsset.fields.file.url}`; - collectionObj.fields.image = _.pick(imageAsset.fields, ['file', 'title']); + coverImageUrl = _.pick(imageAsset.fields, ['file', 'title']); } + const limit = 1000; + const collectionParams = ( + collectionObj + ? { [`fields.${collectionField}.sys.id`]: collectionId } + : {} + ); const params = { content_type: itemType, - [`fields.${collectionField}.sys.id`]: collectionId, + ...collectionParams, order: '-fields.publishedAt', + limit, + skip: 0, }; - const data = await contentfulGet('entries', params, req.pledge); + let items = []; + while (true) { + const data = await contentfulGet('entries', params, req.pledge); + items = items.concat(data.items); + if (data.items.length < limit) { + break; + } + params.skip += limit; + } + const category = 'Religion & Spirituality'; const author = 'The Liturgists Network'; const feed = new RSS({ - title: collectionObj.fields.title, - description: collectionObj.fields.description, + title: title, + description: description, feed_url: getFullRequestUrl(req), site_url: 'https://theliturgists.com', - image_url: getImageUrl(collectionObj), + image_url: coverImageUrl, language: 'en-US', categories: [category], - pubDate: _.get(data.items, [0, 'fields', 'publishedAt'], null), + pubDate: _.get(items, [0, 'fields', 'publishedAt'], null), custom_namespaces: { itunes: 'http://www.itunes.com/dtds/podcast-1.0.dtd', googleplay: 'http://www.google.com/schemas/play-podcasts/1.0', @@ -502,7 +542,7 @@ async function init() { custom_elements: [ { 'itunes:block': 'yes' }, { 'googleplay:block': 'yes' }, - { 'itunes:summary': striptags(collectionObj.fields.description) }, + { 'itunes:summary': striptags(description) }, { 'itunes:author': author }, { 'itunes:owner': [ @@ -512,7 +552,7 @@ async function init() { }, { 'itunes:image': { - _attr: { href: getImageUrl(collectionObj) }, + _attr: { href: coverImageUrl }, }, }, { @@ -524,7 +564,7 @@ async function init() { }); // TODO: handle contentful pagination - data.items.forEach((entry) => { + items.forEach((entry) => { feed.item({ guid: entry.sys.id, title: entry.fields.title, From 4a073763f37937b768405eeac9d88b5e36097381 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 22 May 2019 09:57:47 -0400 Subject: [PATCH 15/52] Remove extra logging and old comments --- index.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/index.js b/index.js index 900285c..347ccd9 100644 --- a/index.js +++ b/index.js @@ -86,7 +86,6 @@ async function getPledge(patreon) { } function filterEntry(entry, pledge, podcasts) { - console.log(entry, podcasts); if (canAccess(pledge, entry, podcasts)) { return _.set(entry, 'fields.patronsOnly', false); } @@ -180,7 +179,6 @@ function handleLiturgistsToken() { // Assign token mapping and pledge to request object for later use req.tokenMapping = items[0].attrs; req.pledge = await getPledge({ token: patreonToken, campaignUrl }); - console.log('pledge:', req.pledge); } catch (err) { console.error(err); res.status(401).json({ error: 'invalid token' }); @@ -269,8 +267,6 @@ async function init() { .exec() .promise(); - console.log('TokenMapping resp:', resp); - const [{ Items: items }] = resp; console.log(`Destroying ${items.length} existing patreon records`); await Promise.all(items.map(item => item.destroy())); @@ -302,7 +298,6 @@ async function init() { async (req, res) => { const path = req.params[1]; // second wildcard match const url = `${patreonApiUrl}/api/${path}`; - console.log('tokenMapping:', req.tokenMapping); const token = req.tokenMapping.patreonToken; const patreonRes = await axios.get( url, @@ -313,7 +308,6 @@ async function init() { }, } ); - console.log('patreon response', patreonRes); res.status(patreonRes.status).json(patreonRes.data); }, ), @@ -563,7 +557,6 @@ async function init() { ], }); - // TODO: handle contentful pagination items.forEach((entry) => { feed.item({ guid: entry.sys.id, From dda5f16dc3110c133084249dcba6c90d60ae69b0 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 22 May 2019 10:00:37 -0400 Subject: [PATCH 16/52] Reuse existing token mapping and tidy up fields --- index.js | 73 ++++++++++++++++++++++++++++++++++++----------- package-lock.json | 19 ++++++++++++ package.json | 1 + 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/index.js b/index.js index 347ccd9..3bfd2b4 100644 --- a/index.js +++ b/index.js @@ -13,6 +13,7 @@ const striptags = require('striptags'); const uuidv4 = require('uuid/v4'); const jwt = require('jsonwebtoken'); const moment = require('moment'); +const urlParse = require('url-parse'); const { getCreds } = require('./src/creds'); const TokenMapping = require('./src/TokenMapping'); @@ -268,20 +269,41 @@ async function init() { .promise(); const [{ Items: items }] = resp; - console.log(`Destroying ${items.length} existing patreon records`); - await Promise.all(items.map(item => item.destroy())); - - const userId = uuidv4(); - await TokenMapping.create({ - userId, - patreonUserId, - patreonToken, - refreshToken, - expiresAt, - }); + if (items.length > 0) { + console.log('Patreon has existing token mapping; updating it'); + if (items.length > 1) { + // shouldn't happen, but tidy up if it does + console.log(`Destroying ${items.length - 1} duplicate mappings`); + await Promise.all(items.slice(1).map(item => item.destroy())); + } + userId = items[0].attrs.userId; + await TokenMapping.update({ + userId, + patreonUserId, + patreonToken, + refreshToken, + expiresAt, + }) + } else { + console.log('Patreon has no existing token mapping; creating one'); + userId = uuidv4(); + + await TokenMapping.create({ + userId, + patreonUserId, + patreonToken, + refreshToken, + expiresAt, + }); + } const { secret } = await getCreds('jwt'); - const token = jwt.sign({ userId }, secret); + + // using noTimestamp here makes the token (and thus URLs containing it) + // a bit shorter, giving us some headroom with podcast clients like + // Overcast which have a hard limit of 256 chars on feed URL length. + // Plus, we weren't using the issuedAt ('iat') timestamp anyway. + const token = jwt.sign({ userId }, secret, { noTimestamp: true }); const data = { liturgistsToken: token, }; @@ -412,6 +434,19 @@ async function init() { return url; } + function getMimeType(entry) { + // dumb assumption of MIME type from filename extension. + // TODO: determine via HTTP HEAD Content-type response header? + const url = getMediaUrl(entry); + const parsed = urlParse(url); + const extension = parsed.pathname.split('.').slice(-1)[0]; + const types = { + mp3: 'mpeg', + }; + const type = _.get(types, extension, extension); + return `audio/${type}`; + } + function imageElementIfDefined(entry) { const href = getImageUrl(entry); if (!href) { @@ -426,7 +461,9 @@ async function init() { function getFullRequestUrl(req) { const { hostname, path } = req; const prefix = /execute-api/.test(hostname) ? `/${stage}` : ''; - return `https://${hostname}${prefix}${path}`; + const query = qs.stringify(req.query); + const querySuffix = query ? `?${query}` : ''; + return `https://${hostname}${prefix}${path}${querySuffix}`; } app.get( @@ -471,6 +508,9 @@ async function init() { let coverImageUrl, title, description; if (collectionObj) { ({ title, description } = collectionObj.fields); + if (collectionObj.type === 'meditationCategory') { + title = `The Liturgists - Meditations - ${title}`; + } if (collectionObj.fields.feedUrl) { res.redirect(collectionObj.fields.feedUrl); @@ -486,7 +526,7 @@ async function init() { coverImageUrl = getImageUrl(collectionObj); } else { // this is the "All Meditations" pseudo-collection - title = 'All Meditations'; + title = 'The Liturgists - Meditations'; description = 'All meditations in all categories.'; const assetId = '4fw1cG2nsTZ9Upl3jpWDVH'; // ID for the cover image @@ -565,7 +605,7 @@ async function init() { date: entry.fields.publishedAt, enclosure: { url: getMediaUrl(entry), - type: 'audio/mpeg', + type: getMimeType(entry), }, image_url: getImageUrl(entry), custom_elements: [ @@ -585,7 +625,8 @@ async function init() { res.set('Content-type', 'application/rss+xml'); res.status(200).send(xml); }, - )); + ), + ); app.use( // eslint-disable-next-line diff --git a/package-lock.json b/package-lock.json index 08f0119..50f1ffe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3864,6 +3864,11 @@ "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" }, + "querystringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz", + "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA==" + }, "range-parser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", @@ -4012,6 +4017,11 @@ "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + }, "resolve": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz", @@ -5141,6 +5151,15 @@ "querystring": "0.2.0" } }, + "url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "url-parse-lax": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", diff --git a/package.json b/package.json index 94907ae..9f4f896 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "serverless": "^1.42.1", "serverless-http": "^1.9.0", "striptags": "^3.1.1", + "url-parse": "^1.4.7", "uuid": "^3.3.2", "yargs": "^13.2.2" }, From 1bb9d396f382d62197b3e635478418ba82559a2f Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Thu, 23 May 2019 22:18:50 -0400 Subject: [PATCH 17/52] Implement token refresh and deletion --- index.js | 158 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 40 deletions(-) diff --git a/index.js b/index.js index 3bfd2b4..eb8c78a 100644 --- a/index.js +++ b/index.js @@ -139,6 +139,50 @@ const patreonAuthUrl = `${patreonBaseUrl}/oauth2/authorize`; const patreonApiUrl = `${patreonBaseUrl}/api/oauth2`; const patreonTokenUrl = `${patreonApiUrl}/token`; +async function refreshPatreonToken(tokenMapping) { + // eslint-disable-next-line camelcase + const { client_id, client_secret } = await getCreds('patreon'); + + const url = patreonTokenUrl; + + console.log(`Refreshing patreon token for user ${tokenMapping.patreonUserId}`); + + const obj = { + grant_type: 'refresh_token', + refresh_token: tokenMapping.refreshToken, + client_id, + client_secret, + }; + const body = qs.stringify(obj); + let patreonRes; + try { + patreonRes = await axios.post(url, body); + console.log('Successfully refreshed patreon token'); + } catch (err) { + Sentry.captureException(err); + throw new Error('Failed to refresh patreon token'); + } + + const { + access_token: patreonToken, + refresh_token: refreshToken, + expires_in: expiresIn, + } = patreonRes.data; + + const expiresAt = moment().add(expiresIn, 'seconds').toISOString(); + + const patreonUser = await getPatreonUser(patreonToken); + const patreonUserId = patreonUser.id; + + await upsertTokenMapping({ + patreonUserId, + patreonToken, + refreshToken, + expiresAt, + }); + return patreonToken; +} + /** * Return a route handler that maps the liturgists token (if present) * to the stored Patreon token, retrieves the patron's pledge (if any), @@ -158,6 +202,9 @@ function handleLiturgistsToken() { _.get(req.query, 'token') ); + let tokenMappingObj; + let tokenMapping; + if (token) { const { secret } = await getCreds('jwt'); try { @@ -170,18 +217,36 @@ function handleLiturgistsToken() { if (!items || items.length === 0) { throw new Error(`no token mapping found for userId ${userId}`); } - console.log('token mapping:', items[0].attrs); - const { patreonToken } = items[0].attrs; - if (!patreonToken) { + tokenMappingObj = items[0]; + tokenMapping = tokenMappingObj.attrs; + let { patreonToken, refreshToken, expiresAt } = tokenMapping; + if (!patreonToken || !refreshToken) { throw new Error(`no patreon token found for userId ${userId}`); } const { campaign_url: campaignUrl } = await getCreds('patreon'); + // if the token is expiring soon, refresh it first + if (moment(expiresAt).isBefore(moment().add(1, 'day'))) { + patreonToken = await refreshPatreonToken(tokenMapping); + } + // Assign token mapping and pledge to request object for later use req.tokenMapping = items[0].attrs; - req.pledge = await getPledge({ token: patreonToken, campaignUrl }); + try { + req.pledge = await getPledge({ token: patreonToken, campaignUrl }); + } catch(err) { + // try to refresh in case the token has expired + const newPatreonToken = await refreshPatreonToken(tokenMapping); + req.pledge = await getPledge({ token: newPatreonToken, campaignUrl }); + } } catch (err) { - console.error(err); + if (tokenMappingObj) { + console.log( + `Invalid token for patreon user ${tokenMapping.patreonUserId}; ` + + 'removing mapping' + ); + await tokenMappingObj.destroy(); + } res.status(401).json({ error: 'invalid token' }); return; } @@ -192,6 +257,40 @@ function handleLiturgistsToken() { ); } +async function upsertTokenMapping(tokenMapping) { + const newTokenMapping = { ...tokenMapping }; + const { patreonUserId } = newTokenMapping; + + const resp = await TokenMapping + .query(patreonUserId) + .usingIndex('patreonUserIdIndex') + .exec() + .promise(); + + const [{ Items: items }] = resp; + if (items.length > 0) { + console.log('Patreon has existing token mapping; updating it'); + if (items.length > 1) { + // shouldn't happen, but tidy up if it does + console.log(`Destroying ${items.length - 1} duplicate mappings`); + await Promise.all(items.slice(1).map(item => item.destroy())); + } + newTokenMapping.userId = items[0].attrs.userId; + await TokenMapping.update(newTokenMapping) + } else { + console.log('Patreon has no existing token mapping; creating one'); + newTokenMapping.userId = uuidv4(); + + await TokenMapping.create(newTokenMapping); + } + + return newTokenMapping; +} + +function redactUrl(url) { + return url.replace(/token=[^&]+/, 'token='); +} + async function init() { const sentry = await getCreds('sentry'); @@ -206,7 +305,14 @@ async function init() { app.use(Sentry.Handlers.requestHandler()); app.use(Sentry.Handlers.errorHandler()); - app.use(morgan('combined')); + app.use(morgan((tokens, req, res) => ( + [ + tokens.method(req, res), + redactUrl(tokens.url(req, res)), + tokens.status(req, res), + tokens['response-time'](req, res), 'ms' + ].join(' ') + ))); app.use(helmet()); // Initialize OAuth2 flow, redirecting to Patreon with client_id @@ -262,40 +368,12 @@ async function init() { const patreonUser = await getPatreonUser(patreonToken); const patreonUserId = patreonUser.id; - const resp = await TokenMapping - .query(patreonUserId) - .usingIndex('patreonUserIdIndex') - .exec() - .promise(); - - const [{ Items: items }] = resp; - if (items.length > 0) { - console.log('Patreon has existing token mapping; updating it'); - if (items.length > 1) { - // shouldn't happen, but tidy up if it does - console.log(`Destroying ${items.length - 1} duplicate mappings`); - await Promise.all(items.slice(1).map(item => item.destroy())); - } - userId = items[0].attrs.userId; - await TokenMapping.update({ - userId, - patreonUserId, - patreonToken, - refreshToken, - expiresAt, - }) - } else { - console.log('Patreon has no existing token mapping; creating one'); - userId = uuidv4(); - - await TokenMapping.create({ - userId, - patreonUserId, - patreonToken, - refreshToken, - expiresAt, - }); - } + const { userId } = await upsertTokenMapping({ + patreonUserId, + patreonToken, + refreshToken, + expiresAt, + }); const { secret } = await getCreds('jwt'); From 9e7b8f5baebfcde817dd9b5fa1dfbd3ef522a772 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 24 May 2019 09:19:32 -0400 Subject: [PATCH 18/52] Set error code indicating Patreon reconnect is required --- index.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index eb8c78a..37559c5 100644 --- a/index.js +++ b/index.js @@ -196,6 +196,16 @@ async function refreshPatreonToken(tokenMapping) { function handleLiturgistsToken() { return wrapAsync( async (req, res, next) => { + const ERROR_CODE_NEED_PATREON_REAUTH = 'needPatreonReauth'; + if (_.has(req.headers, 'x-theliturgists-patreon-token')) { + // old version of the app; needs patreon re-auth + res.status(401).json({ + error: 'invalid auth header', + code: ERROR_CODE_NEED_PATREON_REAUTH, + }); + return; + } + const token = _.get( req.headers, 'x-theliturgists-token', @@ -247,7 +257,10 @@ function handleLiturgistsToken() { ); await tokenMappingObj.destroy(); } - res.status(401).json({ error: 'invalid token' }); + res.status(401).json({ + error: 'invalid token', + code: ERROR_CODE_NEED_PATREON_REAUTH, + }); return; } } From e4ace6493c4c6d125373f82e33ff3192c7367b44 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 27 May 2019 15:55:36 -0400 Subject: [PATCH 19/52] Check pledge amount rather than name Closes #8. --- index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 37559c5..a922400 100644 --- a/index.js +++ b/index.js @@ -31,11 +31,12 @@ function canAccessPodcast(pledge, podcast) { } function canAccessPatronMedia(pledge) { - // Patrons with the 'Master Meditations' reward tier + // Patrons with the $5 reward tier or above // get access to both Meditations and Liturgies // in addition to patrons-only podcasts. - const title = _.get(pledge, 'reward.title'); - return title && /Meditations/i.test(title); + const minMeditationsPledgeCents = 500; + const amount = _.get(pledge, 'amount_cents', 0); + return amount >= minMeditationsPledgeCents; } function canAccess(pledge, item, podcasts) { From 0ff20f42e0f525b6691a77a9d4f6feea13ae3829 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 27 May 2019 16:05:45 -0400 Subject: [PATCH 20/52] Use nanoid rather than jsonwebtoken Much shorter URL; still lots of entropy. --- index.js | 16 ++++----- package-lock.json | 89 +++-------------------------------------------- package.json | 2 +- 3 files changed, 13 insertions(+), 94 deletions(-) diff --git a/index.js b/index.js index a922400..d5fa110 100644 --- a/index.js +++ b/index.js @@ -10,8 +10,7 @@ const Sentry = require('@sentry/node'); const { patreon: patreonAPI } = require('patreon'); const RSS = require('rss'); const striptags = require('striptags'); -const uuidv4 = require('uuid/v4'); -const jwt = require('jsonwebtoken'); +const generate = require('nanoid/generate'); const moment = require('moment'); const urlParse = require('url-parse'); @@ -191,7 +190,7 @@ async function refreshPatreonToken(tokenMapping) { * to the next function. * * Can be used in any route that supplies the `x-theliturgists-token` - * header with the JWT that was returned from the shimmed Patreon + * header with the token that was returned from the shimmed Patreon * OAuth flow. */ function handleLiturgistsToken() { @@ -217,9 +216,8 @@ function handleLiturgistsToken() { let tokenMapping; if (token) { - const { secret } = await getCreds('jwt'); try { - const { userId } = jwt.verify(token, secret); + userId = token; const resp = await TokenMapping .query(userId) .exec() @@ -293,7 +291,8 @@ async function upsertTokenMapping(tokenMapping) { await TokenMapping.update(newTokenMapping) } else { console.log('Patreon has no existing token mapping; creating one'); - newTokenMapping.userId = uuidv4(); + const alphabet = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; + newTokenMapping.userId = generate(alphabet, 32); await TokenMapping.create(newTokenMapping); } @@ -389,13 +388,12 @@ async function init() { expiresAt, }); - const { secret } = await getCreds('jwt'); - // using noTimestamp here makes the token (and thus URLs containing it) // a bit shorter, giving us some headroom with podcast clients like // Overcast which have a hard limit of 256 chars on feed URL length. // Plus, we weren't using the issuedAt ('iat') timestamp anyway. - const token = jwt.sign({ userId }, secret, { noTimestamp: true }); + + const token = userId; const data = { liturgistsToken: token, }; diff --git a/package-lock.json b/package-lock.json index 50f1ffe..29705dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -608,11 +608,6 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, - "buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" - }, "buffer-fill": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", @@ -1308,14 +1303,6 @@ } } }, - "ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -2849,23 +2836,6 @@ "graceful-fs": "^4.1.6" } }, - "jsonwebtoken": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", - "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", - "requires": { - "jws": "^3.2.2", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1", - "semver": "^5.6.0" - } - }, "jszip": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.2.1.tgz", @@ -2877,25 +2847,6 @@ "set-immediate-shim": "~1.0.1" } }, - "jwa": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", - "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", - "requires": { - "jwa": "^1.4.1", - "safe-buffer": "^5.0.1" - } - }, "jwt-decode": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-2.2.0.tgz", @@ -2988,41 +2939,6 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" }, - "lodash.includes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", - "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" - }, - "lodash.isboolean": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", - "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" - }, - "lodash.isinteger": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", - "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" - }, - "lodash.isnumber": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", - "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" - }, - "lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" - }, - "lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" - }, - "lodash.once": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" - }, "lodash.pad": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.5.1.tgz", @@ -3279,6 +3195,11 @@ "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==", "optional": true }, + "nanoid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-2.0.3.tgz", + "integrity": "sha512-NbaoqdhIYmY6FXDRB4eYtDVC9Z9eCbn8TyaiC16LNKtpPv/aqa0tOPD8y6gNE4yUNnaZ7LLhYtXOev/6+cBtfw==" + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", diff --git a/package.json b/package.json index 9f4f896..97c2e7c 100644 --- a/package.json +++ b/package.json @@ -21,10 +21,10 @@ "express": "^4.16.4", "helmet": "^3.15.1", "json-cycle": "^1.3.0", - "jsonwebtoken": "^8.5.1", "lodash": "^4.17.11", "moment": "^2.24.0", "morgan": "^1.9.1", + "nanoid": "^2.0.3", "patreon": "^0.4.1", "progress": "^2.0.3", "qs": "^6.6.0", From 2a77cafeb7141613bec411a0dc639cfae2cd735b Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 27 May 2019 22:02:28 -0400 Subject: [PATCH 21/52] Fix collection type detection --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index d5fa110..93dd3af 100644 --- a/index.js +++ b/index.js @@ -598,7 +598,7 @@ async function init() { let coverImageUrl, title, description; if (collectionObj) { ({ title, description } = collectionObj.fields); - if (collectionObj.type === 'meditationCategory') { + if (collectionObj.sys.contentType.sys.id === 'meditationCategory') { title = `The Liturgists - Meditations - ${title}`; } From 234a5fbb8e4809cee9c31487b1c92366b7f57d76 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 5 Jun 2019 00:09:43 -0400 Subject: [PATCH 22/52] Don't sync fields unless RSS item is newer than Contentful entry --- README.md | 13 +++++++++++++ src/sync.js | 26 ++++++++++++++++++++------ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0017257..245c9c1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,19 @@ Description forthcoming. +## Podbean Feed Sync + +The sync lambda periodically syncs all podcasts in Contentful that +have a feed URL defined. If an episode's publication date is newer +in the feed than in Contentful, the feed data will be treated as +the correct version. However, this will only ever create or update fields; +it will never remove them. + +Note that, with Podbean, the publication date of an item doesn't appear +to be updated unless the item is first saved as a draft, then published. +If an edit is made and published without this intermediate step, the +`pubDate` in the feed doesn't get updated. + ## Dev setup 1. Obtain and configure AWS credentials diff --git a/src/sync.js b/src/sync.js index 543b9c5..98ea042 100644 --- a/src/sync.js +++ b/src/sync.js @@ -4,6 +4,7 @@ const Parser = require('rss-parser'); const yargs = require('yargs'); const ProgressBar = require('progress'); const Sentry = require('@sentry/node'); +const moment = require('moment'); const { getCreds } = require('./creds'); @@ -37,7 +38,7 @@ async function getEpisodes(environment, podcast) { } async function parseFeed(feedUrl) { - const parser = new Parser({ timeout: 5000 }); + const parser = new Parser({ timeout: 30000 }); return parser.parseURL(feedUrl); } @@ -217,11 +218,24 @@ async function syncEpisode( let status = 'unchanged'; if (!_.isEqual(existingEpisode.fields, episodeJson.fields)) { - logProgress(progress, `Updating episode: "${title}"`); - status = 'updated'; - existingEpisode.fields = episodeJson.fields; - existingEpisode = await existingEpisode.update(); - await existingEpisode.publish(); + // only update if the RSS item has been published + // more recently than the Contentful entry + if ( + !existingEpisode.sys.updatedAt || + moment(rssItem.isoDate).isAfter( + moment(existingEpisode.sys.updatedAt) + ) + ) { + logProgress(progress, `Updating episode: "${title}"`); + status = 'updated'; + // only update fields that are explicitly set in the RSS item + existingEpisode.fields = { + ...existingEpisode.fields, + ...episodeJson.fields, + }; + existingEpisode = await existingEpisode.update(); + await existingEpisode.publish(); + } } return { episode: existingEpisode, status }; From c776c3b3001a7f625c5dd757172d072590e3c688 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 7 Jun 2019 08:11:02 -0400 Subject: [PATCH 23/52] Fix image URL property name --- src/sync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync.js b/src/sync.js index 98ea042..d2186aa 100644 --- a/src/sync.js +++ b/src/sync.js @@ -186,7 +186,7 @@ async function syncEpisode( } [ - ['imageUrl', 'itunes.image.href'], + ['imageUrl', 'itunes.image'], ['mediaUrl', 'enclosure.url'], ['duration', 'itunes.duration'], ].forEach(args => setIfPresent(...args)); From 299cbbd5fbe862473be8a6ad78273b5bcbc27cb8 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 7 Jun 2019 08:11:15 -0400 Subject: [PATCH 24/52] Improve sync for different RSS/Contentful fields - The feed is the source of truth for all fields, but... - If any field isn't set in the feed, don't update it in Contentful - Thereby making Contentful the fallback source of truth --- src/sync.js | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/sync.js b/src/sync.js index d2186aa..c6b455d 100644 --- a/src/sync.js +++ b/src/sync.js @@ -217,25 +217,19 @@ async function syncEpisode( } let status = 'unchanged'; - if (!_.isEqual(existingEpisode.fields, episodeJson.fields)) { - // only update if the RSS item has been published - // more recently than the Contentful entry - if ( - !existingEpisode.sys.updatedAt || - moment(rssItem.isoDate).isAfter( - moment(existingEpisode.sys.updatedAt) - ) - ) { - logProgress(progress, `Updating episode: "${title}"`); - status = 'updated'; - // only update fields that are explicitly set in the RSS item - existingEpisode.fields = { - ...existingEpisode.fields, - ...episodeJson.fields, - }; - existingEpisode = await existingEpisode.update(); - await existingEpisode.publish(); - } + + // only update fields that are explicitly set in the RSS item + const newFields = { + ...existingEpisode.fields, + ...episodeJson.fields, + }; + + if (!_.isEqual(existingEpisode.fields, newFields)) { + logProgress(progress, `Updating episode: "${title}"`); + status = 'updated'; + existingEpisode.fields = newFields; + existingEpisode = await existingEpisode.update(); + await existingEpisode.publish(); } return { episode: existingEpisode, status }; From b2421e86b81964460941ca98ba43f3d082035280 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 24 Jul 2019 22:23:24 -0400 Subject: [PATCH 25/52] Add test-notify command --- package-lock.json | 1156 ++++++++++++++++++++++++++- package.json | 4 +- src/notify-data/podcast.json | 75 ++ src/notify-data/podcastEpisode.json | 62 ++ src/notify.js | 78 ++ 5 files changed, 1372 insertions(+), 3 deletions(-) create mode 100644 src/notify-data/podcast.json create mode 100644 src/notify-data/podcastEpisode.json create mode 100644 src/notify.js diff --git a/package-lock.json b/package-lock.json index 29705dd..4917c15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,334 @@ "is-buffer": "^1.1.5" } }, + "@firebase/app": { + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/@firebase/app/-/app-0.4.9.tgz", + "integrity": "sha512-M1An/Id2ozNWbEeanLmczqgu7nS7Qq62u8yjdGOuePhJNie61/10zwPNETGKW/M+sYD6JaZYIsIhP2bQF9ZoDQ==", + "requires": { + "@firebase/app-types": "0.4.0", + "@firebase/logger": "0.1.17", + "@firebase/util": "0.2.20", + "dom-storage": "2.1.0", + "tslib": "1.9.3", + "xmlhttprequest": "1.8.0" + } + }, + "@firebase/app-types": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@firebase/app-types/-/app-types-0.4.0.tgz", + "integrity": "sha512-8erNMHc0V26gA6Nj4W9laVrQrXHsj9K2TEM7eL2IQogGSHLL4vet3UNekYfcGQ2cjfvwUjMzd+BNS/8S7GnfiA==" + }, + "@firebase/database": { + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/@firebase/database/-/database-0.4.6.tgz", + "integrity": "sha512-EpH5JUybuebVzUK1Z1wQ33Hjs8ZJbM6pyAlHDgWcqe4c7hNsHK8QNIxbQXHVfjCtu+EBneFM3MlYF5cWka5Kzw==", + "requires": { + "@firebase/database-types": "0.4.0", + "@firebase/logger": "0.1.17", + "@firebase/util": "0.2.20", + "faye-websocket": "0.11.3", + "tslib": "1.9.3" + } + }, + "@firebase/database-types": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@firebase/database-types/-/database-types-0.4.0.tgz", + "integrity": "sha512-2piRYW7t+2s/P1NPpcI/3+8Y5l2WnJhm9KACoXW5zmoAPlya8R1aEaR2dNHLNePTMHdg04miEDD9fEz4xUqzZA==" + }, + "@firebase/logger": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/@firebase/logger/-/logger-0.1.17.tgz", + "integrity": "sha512-vCuurlKhvEjN5SGbIGfHhVhMsRM6RknAjbEKbT6CgmD6fUnNH2oE1MwbafBK3nLc7i9sQFwZUU/fi4P0Nu/McQ==" + }, + "@firebase/util": { + "version": "0.2.20", + "resolved": "https://registry.npmjs.org/@firebase/util/-/util-0.2.20.tgz", + "integrity": "sha512-Cu5T7RFV54eZdToPXcRKwn7rB0hImbkvLdAmno6mKkoV5s0xDgo9K0PBvftqp8Gg2aDR/B5p+ZjR6xDiSQ42sA==", + "requires": { + "tslib": "1.9.3" + } + }, + "@google-cloud/common": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", + "integrity": "sha512-bLdPzFvvBMtVkwsoBtygE9oUm3yrNmPa71gvOgucYI/GqvNP2tb6RYsDHPq98kvignhcgHGDI5wyNgxaCo8bKQ==", + "optional": true, + "requires": { + "@google-cloud/projectify": "^0.3.3", + "@google-cloud/promisify": "^0.4.0", + "@types/request": "^2.48.1", + "arrify": "^2.0.0", + "duplexify": "^3.6.0", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^3.1.1", + "pify": "^4.0.1", + "retry-request": "^4.0.0", + "teeny-request": "^3.11.3" + }, + "dependencies": { + "gaxios": { + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", + "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", + "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", + "optional": true, + "requires": { + "gaxios": "^1.0.2", + "json-bigint": "^0.3.0" + } + }, + "google-auth-library": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", + "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", + "optional": true, + "requires": { + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^1.2.1", + "gcp-metadata": "^1.0.0", + "gtoken": "^2.3.2", + "https-proxy-agent": "^2.2.1", + "jws": "^3.1.5", + "lru-cache": "^5.0.0", + "semver": "^5.5.0" + } + }, + "google-p12-pem": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", + "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", + "optional": true, + "requires": { + "node-forge": "^0.8.0", + "pify": "^4.0.0" + } + }, + "gtoken": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", + "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", + "optional": true, + "requires": { + "gaxios": "^1.0.4", + "google-p12-pem": "^1.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0", + "pify": "^4.0.0" + } + }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "optional": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==", + "optional": true + }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, + "node-forge": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", + "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==", + "optional": true + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "optional": true + } + } + }, + "@google-cloud/firestore": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-2.2.3.tgz", + "integrity": "sha512-OAaF/2hivinynY0Q0bp2+6rVLzRzMIgNuDEMblZiRYFIos1aBJ1xXZ33RyCecyy+ktm/ARYAVCu+5ZdURitDuw==", + "optional": true, + "requires": { + "bun": "^0.0.12", + "deep-equal": "^1.0.1", + "functional-red-black-tree": "^1.0.1", + "google-gax": "^1.1.2", + "through2": "^3.0.0" + } + }, + "@google-cloud/paginator": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-0.2.0.tgz", + "integrity": "sha512-2ZSARojHDhkLvQ+CS32K+iUhBsWg3AEw+uxtqblA7xoCABDyhpj99FPp35xy6A+XlzMhOSrHHaxFE+t6ZTQq0w==", + "optional": true, + "requires": { + "arrify": "^1.0.1", + "extend": "^3.0.1", + "split-array-stream": "^2.0.0", + "stream-events": "^1.0.4" + }, + "dependencies": { + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "optional": true + } + } + }, + "@google-cloud/projectify": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", + "integrity": "sha512-7522YHQ4IhaafgSunsFF15nG0TGVmxgXidy9cITMe+256RgqfcrfWphiMufW+Ou4kqagW/u3yxwbzVEW3dk2Uw==", + "optional": true + }, + "@google-cloud/promisify": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", + "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" + }, + "@google-cloud/storage": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-2.5.0.tgz", + "integrity": "sha512-q1mwB6RUebIahbA3eriRs8DbG2Ij81Ynb9k8hMqTPkmbd8/S6Z0d6hVvfPmnyvX9Ej13IcmEYIbymuq/RBLghA==", + "optional": true, + "requires": { + "@google-cloud/common": "^0.32.0", + "@google-cloud/paginator": "^0.2.0", + "@google-cloud/promisify": "^0.4.0", + "arrify": "^1.0.0", + "async": "^2.0.1", + "compressible": "^2.0.12", + "concat-stream": "^2.0.0", + "date-and-time": "^0.6.3", + "duplexify": "^3.5.0", + "extend": "^3.0.0", + "gcs-resumable-upload": "^1.0.0", + "hash-stream-validation": "^0.2.1", + "mime": "^2.2.0", + "mime-types": "^2.0.8", + "onetime": "^5.1.0", + "pumpify": "^1.5.1", + "snakeize": "^0.1.0", + "stream-events": "^1.0.1", + "teeny-request": "^3.11.3", + "through2": "^3.0.0", + "xdg-basedir": "^3.0.0" + }, + "dependencies": { + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "optional": true + }, + "async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", + "optional": true, + "requires": { + "lodash": "^4.17.11" + } + }, + "concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "optional": true, + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==", + "optional": true + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "optional": true + }, + "onetime": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", + "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", + "optional": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "optional": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "@grpc/grpc-js": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-0.4.3.tgz", + "integrity": "sha512-09qiFMBh90YZ4P5RFzvpSUvBi9DmftvTaP+mmmTzigps0It5YxuwQNqDAo9pI7SWom/6A5ybxv2CUGNk86+FCg==", + "optional": true, + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz", + "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A==", + "optional": true + } + } + }, + "@grpc/proto-loader": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.1.tgz", + "integrity": "sha512-3y0FhacYAwWvyXshH18eDkUI40wT/uGio7MAegzY8lO5+wVsc19+1A7T0pPptae4kl7bdITL+0cHpnAPmryBjQ==", + "optional": true, + "requires": { + "lodash.camelcase": "^4.3.0", + "protobufjs": "^6.8.6" + } + }, "@hapi/address": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.0.0.tgz", @@ -61,6 +389,60 @@ "@hapi/hoek": "6.x.x" } }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, "@sentry/core": { "version": "4.6.4", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-4.6.4.tgz", @@ -132,11 +514,62 @@ "tslib": "^1.9.3" } }, + "@types/caseless": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", + "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==", + "optional": true + }, + "@types/form-data": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==", + "optional": true, + "requires": { + "@types/node": "*" + } + }, + "@types/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", + "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==" + }, + "@types/node": { + "version": "8.10.50", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.50.tgz", + "integrity": "sha512-+ZbcUwJdaBgOZpwXeT0v+gHC/jQbEfzoc9s4d0rN0JIKeQbuTrT+A2n1aQY6LpZjrLXJT7avVUqiCecCJeeZxA==" + }, + "@types/request": { + "version": "2.48.1", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz", + "integrity": "sha512-ZgEZ1TiD+KGA9LiAAPPJL68Id2UWfeSO62ijSXZjFJArVV+2pKcsVHmrcu+1oiE3q6eDGiFiSolRc4JHoerBBg==", + "optional": true, + "requires": { + "@types/caseless": "*", + "@types/form-data": "*", + "@types/node": "*", + "@types/tough-cookie": "*" + } + }, "@types/stack-trace": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/stack-trace/-/stack-trace-0.0.29.tgz", "integrity": "sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g==" }, + "@types/tough-cookie": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", + "integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg==", + "optional": true + }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, "accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", @@ -349,6 +782,12 @@ "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=" }, + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "optional": true + }, "assign-symbols": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", @@ -469,6 +908,11 @@ "safe-buffer": "5.1.2" } }, + "bignumber.js": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + }, "bl": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", @@ -608,6 +1052,11 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + }, "buffer-fill": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", @@ -624,6 +1073,41 @@ "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", "dev": true }, + "bun": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/bun/-/bun-0.0.12.tgz", + "integrity": "sha512-Toms18J9DqnT+IfWkwxVTB2EaBprHvjlMWrTIsfX4xbu3ZBqVBwrERU0em1IgtRe04wT+wJxMlKHZok24hrcSQ==", + "optional": true, + "requires": { + "readable-stream": "~1.0.32" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "optional": true + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "optional": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", + "optional": true + } + } + }, "bunyan": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz", @@ -907,6 +1391,23 @@ "readable-stream": "^2.0.0" } }, + "compressible": { + "version": "2.0.17", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.17.tgz", + "integrity": "sha512-BGHeLCK1GV7j1bSmQQAi26X+GgWcTjLr/0tzSvMCl3LH1w1IJ4PFSPoV5316b30cneTziC+B1a+3OjoSUcQYmw==", + "optional": true, + "requires": { + "mime-db": ">= 1.40.0 < 2" + }, + "dependencies": { + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==", + "optional": true + } + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1055,6 +1556,12 @@ "resolved": "https://registry.npmjs.org/dasherize/-/dasherize-2.0.0.tgz", "integrity": "sha1-bYCcnNDPe7iVLYD8hPoT1H3bEwg=" }, + "date-and-time": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.6.3.tgz", + "integrity": "sha512-lcWy3AXDRJOD7MplwZMmNSRM//kZtJaLz4n6D1P5z9wEmZGBKhJRBIr1Xs9KNQJmdXPblvgffynYji4iylUTcA==", + "optional": true + }, "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", @@ -1154,6 +1661,12 @@ } } }, + "deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", + "optional": true + }, "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -1203,6 +1716,14 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "dicer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", + "integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==", + "requires": { + "streamsearch": "0.1.2" + } + }, "dns-prefetch-control": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/dns-prefetch-control/-/dns-prefetch-control-0.1.0.tgz", @@ -1217,6 +1738,11 @@ "esutils": "^2.0.2" } }, + "dom-storage": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/dom-storage/-/dom-storage-2.1.0.tgz", + "integrity": "sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q==" + }, "dont-sniff-mimetype": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.0.0.tgz", @@ -1258,6 +1784,17 @@ "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" }, + "duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "requires": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, "dynamodb": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/dynamodb/-/dynamodb-1.2.0.tgz", @@ -1303,6 +1840,14 @@ } } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1334,6 +1879,12 @@ "once": "^1.4.0" } }, + "ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=", + "optional": true + }, "entities": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", @@ -1791,6 +2342,11 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, "events": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", @@ -1918,6 +2474,19 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, + "fast-text-encoding": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", + "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==" + }, + "faye-websocket": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.3.tgz", + "integrity": "sha512-D2y4bovYpzziGgbHYtGCMjlJM36vAl/y+xUyn1C+FVx8szd1E+86KwVw6XvYSzOP8iMpm1X0I4xJD+QtUb36OA==", + "requires": { + "websocket-driver": ">=0.5.1" + } + }, "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -2013,6 +2582,21 @@ "locate-path": "^2.0.0" } }, + "firebase-admin": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-8.2.0.tgz", + "integrity": "sha512-SiF4ivEknRWvwtFLgUxfxN7kR6/3bcoNd7pXVKPcszW6lHcMXe5qY58MwKIfDTN1JlayBiwkZjealnGZ2G8/Yg==", + "requires": { + "@firebase/app": "^0.4.4", + "@firebase/database": "^0.4.4", + "@google-cloud/firestore": "^2.0.0", + "@google-cloud/storage": "^2.5.0", + "@types/node": "^8.0.53", + "dicer": "^0.3.0", + "jsonwebtoken": "8.1.0", + "node-forge": "0.7.4" + } + }, "flat-cache": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.4.tgz", @@ -2127,8 +2711,7 @@ "functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", - "dev": true + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=" }, "gauge": { "version": "1.2.7", @@ -2142,6 +2725,181 @@ "lodash.padstart": "^4.1.0" } }, + "gaxios": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.0.1.tgz", + "integrity": "sha512-c1NXovTxkgRJTIgB2FrFmOFg4YIV6N/bAa4f/FZ4jIw13Ql9ya/82x69CswvotJhbV3DiGnlTZwoq2NVXk2Irg==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.3.0" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + } + } + }, + "gcp-metadata": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-2.0.1.tgz", + "integrity": "sha512-nrbLj5O1MurvpLC/doFwzdTfKnmYGDYXlY/v7eQ4tJNVIvQXbOK672J9UFbradbtmuTkyHzjpzD8HD0Djz0LWw==", + "optional": true, + "requires": { + "gaxios": "^2.0.0", + "json-bigint": "^0.3.0" + } + }, + "gcs-resumable-upload": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-1.1.0.tgz", + "integrity": "sha512-uBz7uHqp44xjSDzG3kLbOYZDjxxR/UAGbB47A0cC907W6yd2LkcyFDTHg+bjivkHMwiJlKv4guVWcjPCk2zScg==", + "optional": true, + "requires": { + "abort-controller": "^2.0.2", + "configstore": "^4.0.0", + "gaxios": "^1.5.0", + "google-auth-library": "^3.0.0", + "pumpify": "^1.5.1", + "stream-events": "^1.0.4" + }, + "dependencies": { + "abort-controller": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-2.0.3.tgz", + "integrity": "sha512-EPSq5wr2aFyAZ1PejJB32IX9Qd4Nwus+adnp7STYFM5/23nLPBazqZ1oor6ZqbH+4otaaGXTlC8RN5hq3C8w9Q==", + "optional": true, + "requires": { + "event-target-shim": "^5.0.0" + } + }, + "configstore": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-4.0.0.tgz", + "integrity": "sha512-CmquAXFBocrzaSM8mtGPMM/HiWmyIpr4CcJl/rgY2uCObZ/S7cKU0silxslqJejl+t/T9HS8E0PUNQD81JGUEQ==", + "optional": true, + "requires": { + "dot-prop": "^4.1.0", + "graceful-fs": "^4.1.2", + "make-dir": "^1.0.0", + "unique-string": "^1.0.0", + "write-file-atomic": "^2.0.0", + "xdg-basedir": "^3.0.0" + } + }, + "gaxios": { + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", + "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.3.0" + }, + "dependencies": { + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + } + } + }, + "gcp-metadata": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", + "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", + "optional": true, + "requires": { + "gaxios": "^1.0.2", + "json-bigint": "^0.3.0" + } + }, + "google-auth-library": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", + "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", + "optional": true, + "requires": { + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^1.2.1", + "gcp-metadata": "^1.0.0", + "gtoken": "^2.3.2", + "https-proxy-agent": "^2.2.1", + "jws": "^3.1.5", + "lru-cache": "^5.0.0", + "semver": "^5.5.0" + } + }, + "google-p12-pem": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", + "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", + "optional": true, + "requires": { + "node-forge": "^0.8.0", + "pify": "^4.0.0" + } + }, + "gtoken": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", + "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", + "optional": true, + "requires": { + "gaxios": "^1.0.4", + "google-p12-pem": "^1.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0", + "pify": "^4.0.0" + } + }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "optional": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==", + "optional": true + }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, + "node-forge": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", + "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==", + "optional": true + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "optional": true + } + } + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -2209,6 +2967,89 @@ "pinkie-promise": "^2.0.0" } }, + "google-auth-library": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-4.2.5.tgz", + "integrity": "sha512-Vfsr82M1KTdT0H0wjawwp0LHsT6mPKSolRp21ZpJ7Ydq63zRe8DbGKjRCCrhsRZHg+p17DuuSCMEznwk3CJRdw==", + "optional": true, + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^2.0.0", + "gcp-metadata": "^2.0.0", + "gtoken": "^3.0.0", + "jws": "^3.1.5", + "lru-cache": "^5.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "optional": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "optional": true + } + } + }, + "google-gax": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.1.4.tgz", + "integrity": "sha512-Us35ZD3T+MKvSpCN6lO+VBH1tDbNwhyOihNnPaCBerVIdkmEhHBk+onPnU84YvhCd6SRRHYt1B2vWZEH5t1SdQ==", + "optional": true, + "requires": { + "@grpc/grpc-js": "^0.4.0", + "@grpc/proto-loader": "^0.5.1", + "duplexify": "^3.6.0", + "google-auth-library": "^4.0.0", + "is-stream-ended": "^0.1.4", + "lodash.at": "^4.6.0", + "lodash.has": "^4.5.2", + "protobufjs": "^6.8.8", + "retry-request": "^4.0.0", + "semver": "^6.0.0", + "walkdir": "^0.4.0" + }, + "dependencies": { + "semver": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz", + "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A==", + "optional": true + }, + "walkdir": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.0.tgz", + "integrity": "sha512-Ps0LSr9doEPbF4kEQi6sk5RgzIGLz9+OroGj1y2osIVnufjNQWSLEGIbZwW5V+j/jK8lCj/+8HSWs+6Q/rnViA==", + "optional": true + } + } + }, + "google-p12-pem": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.1.tgz", + "integrity": "sha512-6h6x+eBX3k+IDSe/c8dVYmn8Mzr1mUcmKC9MdUSwaBkFAXlqBEnwFWmSFgGC+tcqtsLn73BDP/vUNWEehf1Rww==", + "optional": true, + "requires": { + "node-forge": "^0.8.0" + }, + "dependencies": { + "node-forge": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", + "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==", + "optional": true + } + } + }, "got": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", @@ -2245,6 +3086,26 @@ "lodash": "^4.17.5" } }, + "gtoken": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-3.0.2.tgz", + "integrity": "sha512-BOBi6Zz31JfxhSHRZBIDdbwIbOPyux10WxJHdx8wz/FMP1zyN1xFrsAWsgcLe5ww5v/OZu/MePUEZAjgJXSauA==", + "optional": true, + "requires": { + "gaxios": "^2.0.0", + "google-p12-pem": "^2.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0" + }, + "dependencies": { + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==", + "optional": true + } + } + }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -2320,6 +3181,27 @@ } } }, + "hash-stream-validation": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.1.tgz", + "integrity": "sha1-7Mm5l7IYvluzEphii7gHhptz3NE=", + "optional": true, + "requires": { + "through2": "^2.0.0" + }, + "dependencies": { + "through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "optional": true, + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + } + } + }, "helmet": { "version": "3.15.1", "resolved": "https://registry.npmjs.org/helmet/-/helmet-3.15.1.tgz", @@ -2397,6 +3279,11 @@ "statuses": ">= 1.4.0 < 2" } }, + "http-parser-js": { + "version": "0.4.10", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.10.tgz", + "integrity": "sha1-ksnBN0w1CF912zWexWzCV8u5P6Q=" + }, "https-proxy-agent": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", @@ -2708,6 +3595,12 @@ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, + "is-stream-ended": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==", + "optional": true + }, "is-symbol": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", @@ -2780,6 +3673,14 @@ "esprima": "^4.0.0" } }, + "json-bigint": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "requires": { + "bignumber.js": "^7.0.0" + } + }, "json-cycle": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/json-cycle/-/json-cycle-1.3.0.tgz", @@ -2836,6 +3737,23 @@ "graceful-fs": "^4.1.6" } }, + "jsonwebtoken": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.1.0.tgz", + "integrity": "sha1-xjl80uX9WD1lwAeoPce7eOaYK4M=", + "requires": { + "jws": "^3.1.4", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.0.0", + "xtend": "^4.0.1" + } + }, "jszip": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.2.1.tgz", @@ -2847,6 +3765,25 @@ "set-immediate-shim": "~1.0.1" } }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "jwt-decode": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-2.2.0.tgz", @@ -2934,11 +3871,64 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, + "lodash.at": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", + "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g=", + "optional": true + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", + "optional": true + }, "lodash.difference": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" }, + "lodash.has": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", + "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=", + "optional": true + }, + "lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" + }, + "lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" + }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, "lodash.pad": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.5.1.tgz", @@ -2959,6 +3949,11 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", @@ -3259,6 +4254,11 @@ "is-stream": "^1.0.1" } }, + "node-forge": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.4.tgz", + "integrity": "sha512-8Df0906+tq/omxuCZD6PqhPaQDYuyJ1d+VITgxoIA8zvQd1ru+nMJcDChHH324MWitIgbVkAkQoGEEVJNpn/PA==" + }, "node-uuid": { "version": "1.4.8", "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz", @@ -3747,6 +4747,33 @@ "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=" }, + "protobufjs": { + "version": "6.8.8", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "10.14.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", + "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==" + } + } + }, "proxy-addr": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", @@ -3770,6 +4797,27 @@ "once": "^1.3.1" } }, + "pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "requires": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + }, + "dependencies": { + "pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -3977,6 +5025,27 @@ "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" }, + "retry-request": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", + "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==", + "optional": true, + "requires": { + "debug": "^4.1.1", + "through2": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "optional": true, + "requires": { + "ms": "^2.1.1" + } + } + } + }, "rimraf": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", @@ -4340,6 +5409,12 @@ } } }, + "snakeize": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz", + "integrity": "sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0=", + "optional": true + }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -4505,6 +5580,15 @@ "integrity": "sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g==", "dev": true }, + "split-array-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/split-array-stream/-/split-array-stream-2.0.0.tgz", + "integrity": "sha512-hmMswlVY91WvGMxs0k8MRgq8zb2mSen4FmDNc5AFiTWtrBpdZN6nwD6kROVe4vNL+ywrvbCKsWVCnEd4riELIg==", + "optional": true, + "requires": { + "is-stream-ended": "^0.1.4" + } + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", @@ -4598,6 +5682,19 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" }, + "stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "requires": { + "stubs": "^3.0.0" + } + }, + "stream-shift": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", + "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" + }, "stream-to-array": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz", @@ -4634,6 +5731,11 @@ } } }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -4697,6 +5799,11 @@ "resolved": "https://registry.npmjs.org/striptags/-/striptags-3.1.1.tgz", "integrity": "sha1-yMPn/db7S7OjKjt1LltePjgJPr0=" }, + "stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=" + }, "superagent": { "version": "3.8.3", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.3.tgz", @@ -4811,6 +5918,23 @@ "xtend": "^4.0.0" } }, + "teeny-request": { + "version": "3.11.3", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", + "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", + "requires": { + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.2.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + } + } + }, "term-size": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", @@ -4830,6 +5954,14 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "through2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "requires": { + "readable-stream": "2 || 3" + } + }, "timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", @@ -5134,6 +6266,21 @@ "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.0.11.tgz", "integrity": "sha1-oW0CXrkxvQO1LzCMrtD0D86+lTI=" }, + "websocket-driver": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.3.tgz", + "integrity": "sha512-bpxWlvbbB459Mlipc5GBzzZwhoZgGEZLuqPaR0INBGnPAY1vdBX6hPnoFXiw+3yWxDuHyQjO2oXTMyS8A5haFg==", + "requires": { + "http-parser-js": ">=0.4.0 <0.4.11", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + } + }, + "websocket-extensions": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", + "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==" + }, "whatwg-fetch": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz", @@ -5257,6 +6404,11 @@ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" }, + "xmlhttprequest": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", + "integrity": "sha1-Z/4HXFwk/vOfnWX197f+dRcZaPw=" + }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index 97c2e7c..ae1dc20 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "lint": "eslint", "deploy": "sls deploy", "auth-deploy": "npm run auth-sls deploy", - "auth-sls": "assume-role theliturgists-deploy serverless" + "auth-sls": "assume-role theliturgists-deploy serverless", + "test-notify": "node ./src/notify.js" }, "dependencies": { "@hapi/joi": "^15.0.3", @@ -19,6 +20,7 @@ "contentful-management": "^5.7.0", "dynamodb": "^1.2.0", "express": "^4.16.4", + "firebase-admin": "^8.2.0", "helmet": "^3.15.1", "json-cycle": "^1.3.0", "lodash": "^4.17.11", diff --git a/src/notify-data/podcast.json b/src/notify-data/podcast.json new file mode 100644 index 0000000..36607ef --- /dev/null +++ b/src/notify-data/podcast.json @@ -0,0 +1,75 @@ +{ + "sys": { + "space": { + "sys": { + "type": "Link", + "linkType": "Space", + "id": "80rg4ikyq3mh" + } + }, + "id": "5WozxaSfgsSLunMw1F8mkQ", + "type": "Entry", + "createdAt": "2019-03-25T13:05:20.496Z", + "updatedAt": "2019-06-07T12:50:37.135Z", + "environment": { + "sys": { + "id": "master", + "type": "Link", + "linkType": "Environment" + } + }, + "revision": 4, + "contentType": { + "sys": { + "type": "Link", + "linkType": "ContentType", + "id": "podcast" + } + }, + "locale": "en-US" + }, + "fields": { + "title": "Test-Podcast", + "description": "Just for testing.", + "image": { + "sys": { + "space": { + "sys": { + "type": "Link", + "linkType": "Space", + "id": "80rg4ikyq3mh" + } + }, + "id": "5VAWOmxcZ2PWjYx8lnyN2S", + "type": "Asset", + "createdAt": "2019-02-27T22:43:45.356Z", + "updatedAt": "2019-02-27T22:43:45.356Z", + "environment": { + "sys": { + "id": "master", + "type": "Link", + "linkType": "Environment" + } + }, + "revision": 1, + "locale": "en-US" + }, + "fields": { + "title": "Reset, large", + "file": { + "url": "//images.ctfassets.net/80rg4ikyq3mh/5VAWOmxcZ2PWjYx8lnyN2S/f54de8dfa0520674e7cee1d909847101/Untitled", + "details": { + "size": 446398, + "image": { + "width": 1600, + "height": 1200 + } + }, + "fileName": "Untitled", + "contentType": "image/jpeg" + } + } + }, + "patronsOnly": false + } +} diff --git a/src/notify-data/podcastEpisode.json b/src/notify-data/podcastEpisode.json new file mode 100644 index 0000000..d0a116c --- /dev/null +++ b/src/notify-data/podcastEpisode.json @@ -0,0 +1,62 @@ +{ + "sys": { + "type": "Entry", + "id": "4J44GPppdkrDuiiQCjZSwT", + "space": { + "sys": { + "type": "Link", + "linkType": "Space", + "id": "80rg4ikyq3mh" + } + }, + "environment": { + "sys": { + "id": "master", + "type": "Link", + "linkType": "Environment" + } + }, + "contentType": { + "sys": { + "type": "Link", + "linkType": "ContentType", + "id": "podcastEpisode" + } + }, + "revision": 1, + "createdAt": "2019-07-06T01:20:53.302Z", + "updatedAt": "2019-07-06T01:20:53.302Z" + }, + "fields": { + "title": { + "en-US": "Test Episode" + }, + "description": { + "en-US": "asdfasgdsagdsgda" + }, + "media": { + "en-US": { + "sys": { + "type": "Link", + "linkType": "Asset", + "id": "7AcOpTMU6GfKNH0MzJFjaR" + } + } + }, + "duration": { + "en-US": "0:0:4" + }, + "publishedAt": { + "en-US": "2019-07-05T12:00-04:00" + }, + "podcast": { + "en-US": { + "sys": { + "type": "Link", + "linkType": "Entry", + "id": "5WozxaSfgsSLunMw1F8mkQ" + } + } + } + } +} diff --git a/src/notify.js b/src/notify.js new file mode 100644 index 0000000..a8a49f3 --- /dev/null +++ b/src/notify.js @@ -0,0 +1,78 @@ +const _ = require('lodash'); +const firebase = require('firebase-admin'); + +firebase.initializeApp({ + credential: firebase.credential.applicationDefault(), +}); + +const TOPIC_PUBLIC_MEDIA = 'new-public-media'; +const TOPIC_PATRON_PODCAST = 'new-patron-podcast'; +const TOPIC_PATRON_MEDITATION = 'new-patron-meditation'; +const TOPIC_PATRON_LITURGY = 'new-patron-liturgy'; + +function getCollection(entry) { + +} + +function getTopic(entry, collectionEntry) { + if (_.get(entry, 'fields.isFreePreview.en-US')) { + return TOPIC_PUBLIC_MEDIA; + } + if (entry.sys.contentType.sys.id === 'meditation') { + return TOPIC_PATRON_MEDITATION; + } + if (entry.sys.contentType.sys.id === 'liturgy') { + return TOPIC_PATRON_LITURGY; + } + if (entry.sys.contentType.sys.id === 'podcastEpisode') { + // XXX: assumes all patron podcasts have the same minimum pledge + // (which is true at present, but perhaps not forever) + return collectionEntry.fields.minimumPledgeDollars ? TOPIC_PATRON_PODCAST : TOPIC_PUBLIC_MEDIA; + } + return TOPIC_PUBLIC_MEDIA; +} + +function getImageUrl(collectionEntry) { + if (collectionEntry.fields.image) { + return `https:${collectionEntry.fields.image.fields.file.url}` + } + + return collectionEntry.fields.imageUrl; +} + +function makeNotification(entry, collectionEntry) { + const topic = getTopic(entry, collectionEntry); + const title = entry.fields.title['en-US']; + const subtitle = collectionEntry.fields.title; + + return { + topic, + notification: { + title: `${title} (${subtitle})`, + body: entry.fields.description['en-US'], + }, + android: { + notification: { + channel_id: 'main', + }, + }, + data: { + contentType: entry.sys.contentType.sys.id, + entryId: entry.sys.id, + }, + }; +} + +function notifyNewItem(entry, collectionEntry) { + const message = makeNotification(entry, collectionEntry); + return firebase.messaging().send(message) + .then(response => console.log('Successfully sent message: ', response)) + .catch(error => console.error('Error sending message: ', error)); +} + + +if (require.main === module) { + const entry = require('./notify-data/podcastEpisode.json'); + const collectionEntry = require('./notify-data/podcast.json'); + notifyNewItem(entry, collectionEntry).then(() => process.exit(0)); +} From d1612ea6273c7db0730bd4054ff51a2c4443f08f Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 2 Aug 2019 09:43:45 -0400 Subject: [PATCH 26/52] Add Contentful webhook handler that sends Firebase notifications --- index.js | 41 ++++++++++++++++++++++++++++++++++++++++- package.json | 3 +-- serverless.yml | 1 + src/creds.js | 3 +++ src/notify.js | 38 +++++++++++++++++++------------------- 5 files changed, 64 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 93dd3af..ebdd418 100644 --- a/index.js +++ b/index.js @@ -16,6 +16,7 @@ const urlParse = require('url-parse'); const { getCreds } = require('./src/creds'); const TokenMapping = require('./src/TokenMapping'); +const { notifyNewItem } = require('./src/notify'); const stage = process.env.SLS_STAGE; @@ -430,7 +431,8 @@ async function init() { * * @param {string} path contentful resource path (after environment) * @param {object} params contentful query params from request object - * @param {string} patreonToken patreon token from request header + * @param {object} pledge Patreon pledge API JSON + * @param {boolean} filter if false, don't filter by patron pledge (default true) * @returns {object} contentful API response * @throws {Error} on contentful API error * error object has 'status' and 'json' fields @@ -718,6 +720,43 @@ async function init() { ), ); + app.post( + '*/contentful-webhook', + bodyParser.json({ + type: 'application/vnd.contentful.management.v1+json', + }), + wrapAsync( + async (req, res) => { + const entry = req.body; + console.log('Webhook payload: ', JSON.stringify(entry, null, 2)); + const contentType = entry.sys.contentType.sys.id; + const collectionField = { + podcastEpisode: 'podcast', + meditation: 'meditationCategory', + liturgyItem: 'liturgy', + }[contentType]; + const collectionId = entry.fields[collectionField]['en-US'].sys.id; + + try { + const collectionEntry = await contentfulGet( + `entries/${collectionId}`, + {}, + null, + false, + ); + + await notifyNewItem(entry, collectionEntry); + + res.status(200).json({ status: 'success' }); + } catch (e) { + console.error(e); + Sentry.captureException(e); + res.status(500).json({ error: e.message }); + } + } + ), + ); + app.use( // eslint-disable-next-line async (error, req, res, next) => { diff --git a/package.json b/package.json index ae1dc20..92b27bc 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,7 @@ "lint": "eslint", "deploy": "sls deploy", "auth-deploy": "npm run auth-sls deploy", - "auth-sls": "assume-role theliturgists-deploy serverless", - "test-notify": "node ./src/notify.js" + "auth-sls": "assume-role theliturgists-deploy serverless" }, "dependencies": { "@hapi/joi": "^15.0.3", diff --git a/serverless.yml b/serverless.yml index 84d9fe4..32033d4 100644 --- a/serverless.yml +++ b/serverless.yml @@ -67,6 +67,7 @@ functions: - http: GET /patreon/api/{proxy+} - http: GET /contentful/{proxy+} - http: GET /rss/{proxy+} + - http: POST /contentful-webhook sync: handler: src/sync.handler timeout: 300 diff --git a/src/creds.js b/src/creds.js index 02bf944..13cd5cf 100644 --- a/src/creds.js +++ b/src/creds.js @@ -23,6 +23,9 @@ const credSpecs = { jwt: { secret: `/${stage}/JWT_SECRET`, }, + firebase: { + serviceAccount: `/${stage}/FIREBASE_SERVICE_ACCOUNT`, + }, }; module.exports.getCreds = async (name) => { diff --git a/src/notify.js b/src/notify.js index a8a49f3..8bb071f 100644 --- a/src/notify.js +++ b/src/notify.js @@ -1,19 +1,25 @@ const _ = require('lodash'); const firebase = require('firebase-admin'); -firebase.initializeApp({ - credential: firebase.credential.applicationDefault(), -}); +const { getCreds } = require('./creds'); + +let firebaseInitialized = false; + +async function initializeFirebase() { + if (!firebaseInitialized) { + const { serviceAccount } = await getCreds('firebase'); + firebase.initializeApp({ + credential: firebase.credential.cert(JSON.parse(serviceAccount)), + }); + firebaseInitialized = true; + } +} const TOPIC_PUBLIC_MEDIA = 'new-public-media'; const TOPIC_PATRON_PODCAST = 'new-patron-podcast'; const TOPIC_PATRON_MEDITATION = 'new-patron-meditation'; const TOPIC_PATRON_LITURGY = 'new-patron-liturgy'; -function getCollection(entry) { - -} - function getTopic(entry, collectionEntry) { if (_.get(entry, 'fields.isFreePreview.en-US')) { return TOPIC_PUBLIC_MEDIA; @@ -63,16 +69,10 @@ function makeNotification(entry, collectionEntry) { }; } -function notifyNewItem(entry, collectionEntry) { +module.exports.notifyNewItem = async (entry, collectionEntry) => { + await initializeFirebase(); const message = makeNotification(entry, collectionEntry); - return firebase.messaging().send(message) - .then(response => console.log('Successfully sent message: ', response)) - .catch(error => console.error('Error sending message: ', error)); -} - - -if (require.main === module) { - const entry = require('./notify-data/podcastEpisode.json'); - const collectionEntry = require('./notify-data/podcast.json'); - notifyNewItem(entry, collectionEntry).then(() => process.exit(0)); -} + const response = await firebase.messaging().send(message); + console.log(`Successfully sent message to topic "${message.topic}": `, response); + return response; +}; From 1bbfe02a5d08f9da6181e4cac059305baae2709b Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 2 Aug 2019 22:14:50 -0400 Subject: [PATCH 27/52] Fix category field name --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index ebdd418..2630067 100644 --- a/index.js +++ b/index.js @@ -732,7 +732,7 @@ async function init() { const contentType = entry.sys.contentType.sys.id; const collectionField = { podcastEpisode: 'podcast', - meditation: 'meditationCategory', + meditation: 'category', liturgyItem: 'liturgy', }[contentType]; const collectionId = entry.fields[collectionField]['en-US'].sys.id; From 17d04039067c99a751167032f43f90fa54c5b50e Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 3 Aug 2019 15:43:02 -0400 Subject: [PATCH 28/52] Add verification token header check --- index.js | 7 +++++++ src/creds.js | 1 + 2 files changed, 8 insertions(+) diff --git a/index.js b/index.js index 2630067..0e0d8d3 100644 --- a/index.js +++ b/index.js @@ -727,6 +727,13 @@ async function init() { }), wrapAsync( async (req, res) => { + const { webhookVerificationToken } = await getCreds('contentful'); + const token = req.headers['x-theliturgists-webhook-verification-token']; + if (token !== webhookVerificationToken) { + res.status(401).json({ error: 'invalid verification token'}); + return; + } + const entry = req.body; console.log('Webhook payload: ', JSON.stringify(entry, null, 2)); const contentType = entry.sys.contentType.sys.id; diff --git a/src/creds.js b/src/creds.js index 13cd5cf..63f00e0 100644 --- a/src/creds.js +++ b/src/creds.js @@ -13,6 +13,7 @@ const credSpecs = { space: `/${stage}/CONTENTFUL_SPACE`, environment: `/${stage}/CONTENTFUL_ENVIRONMENT`, accessToken: `/${stage}/CONTENTFUL_ACCESS_TOKEN`, + webhookVerificationToken: `/${stage}/CONTENTFUL_WEBHOOK_VERIFICATION_TOKEN`, }, contentfulManagement: { accessToken: `/${stage}/CONTENTFUL_MANAGEMENT_ACCESS_TOKEN`, From 8a24e5712c643a951e63052b436c8309588cc273 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 3 Aug 2019 16:37:37 -0400 Subject: [PATCH 29/52] Only notify for recently-published entries --- index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 0e0d8d3..fb29a17 100644 --- a/index.js +++ b/index.js @@ -735,7 +735,13 @@ async function init() { } const entry = req.body; - console.log('Webhook payload: ', JSON.stringify(entry, null, 2)); + const publishedAt = moment(entry.fields.publishedAt['en-US']); + const oneDayInMillis = 1000 * 60 * 60 * 24; + if (moment().diff(publishedAt) > oneDayInMillis) { + res.status(200).json({ status: 'older than one day; not notifying' }); + return; + } + const contentType = entry.sys.contentType.sys.id; const collectionField = { podcastEpisode: 'podcast', From 84da12f051f4acf9532ee26954abacce7f7e983b Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 3 Aug 2019 16:37:56 -0400 Subject: [PATCH 30/52] Note channel in success response --- index.js | 4 ++-- src/notify.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index fb29a17..1b4615f 100644 --- a/index.js +++ b/index.js @@ -758,9 +758,9 @@ async function init() { false, ); - await notifyNewItem(entry, collectionEntry); + const { message } = await notifyNewItem(entry, collectionEntry); - res.status(200).json({ status: 'success' }); + res.status(200).json({ status: `notified topic ${message.topic}` }); } catch (e) { console.error(e); Sentry.captureException(e); diff --git a/src/notify.js b/src/notify.js index 8bb071f..016b54b 100644 --- a/src/notify.js +++ b/src/notify.js @@ -74,5 +74,5 @@ module.exports.notifyNewItem = async (entry, collectionEntry) => { const message = makeNotification(entry, collectionEntry); const response = await firebase.messaging().send(message); console.log(`Successfully sent message to topic "${message.topic}": `, response); - return response; + return { message, response }; }; From 355042f070a62ed7ca9209621f074b8ca2777f71 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 3 Aug 2019 16:38:11 -0400 Subject: [PATCH 31/52] Limit description size to avoid firebse validation error --- src/notify.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/notify.js b/src/notify.js index 016b54b..9692584 100644 --- a/src/notify.js +++ b/src/notify.js @@ -46,6 +46,14 @@ function getImageUrl(collectionEntry) { return collectionEntry.fields.imageUrl; } +function truncate(str, limit = 1024) { + if (str.length < limit) { + return str; + } + + return str.slice(0, limit - 3) + '...'; +} + function makeNotification(entry, collectionEntry) { const topic = getTopic(entry, collectionEntry); const title = entry.fields.title['en-US']; @@ -55,7 +63,7 @@ function makeNotification(entry, collectionEntry) { topic, notification: { title: `${title} (${subtitle})`, - body: entry.fields.description['en-US'], + body: truncate(entry.fields.description['en-US']), }, android: { notification: { From 932bd83c84f8875ea4ba5fd44df1b54eaeeb564f Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 6 Aug 2019 08:43:23 -0400 Subject: [PATCH 32/52] Strip HTML tags in notification body --- src/notify.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/notify.js b/src/notify.js index 9692584..093627a 100644 --- a/src/notify.js +++ b/src/notify.js @@ -54,6 +54,14 @@ function truncate(str, limit = 1024) { return str.slice(0, limit - 3) + '...'; } +function stripTags(html) { + return html.replace(/<[^>]+>/g, ''); +} + +function formatDescription(description) { + return truncate(stripTags(description)); +} + function makeNotification(entry, collectionEntry) { const topic = getTopic(entry, collectionEntry); const title = entry.fields.title['en-US']; @@ -63,7 +71,7 @@ function makeNotification(entry, collectionEntry) { topic, notification: { title: `${title} (${subtitle})`, - body: truncate(entry.fields.description['en-US']), + body: formatDescription(entry.fields.description['en-US']), }, android: { notification: { From 7d70799e619cc9ec4af3cffed6509b7c01687802 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2019 01:51:33 +0000 Subject: [PATCH 33/52] Bump lodash from 4.17.11 to 4.17.13 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.11 to 4.17.13. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.11...4.17.13) Signed-off-by: dependabot[bot] --- package-lock.json | 92 +++++++++++++++++++++++++++++++++-------------- package.json | 2 +- 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4917c15..11260b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -104,6 +104,7 @@ "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", + "optional": true, "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -179,7 +180,8 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "optional": true }, "node-forge": { "version": "0.8.5", @@ -190,7 +192,8 @@ "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "optional": true }, "yallist": { "version": "3.0.3", @@ -242,7 +245,8 @@ "@google-cloud/promisify": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", - "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" + "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==", + "optional": true }, "@google-cloud/storage": { "version": "2.5.0", @@ -392,27 +396,32 @@ "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=", + "optional": true }, "@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "optional": true }, "@protobufjs/codegen": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "optional": true }, "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=", + "optional": true }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "optional": true, "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -421,27 +430,32 @@ "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=", + "optional": true }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=", + "optional": true }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=", + "optional": true }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=", + "optional": true }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=", + "optional": true }, "@sentry/core": { "version": "4.6.4", @@ -532,7 +546,8 @@ "@types/long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", - "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==" + "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==", + "optional": true }, "@types/node": { "version": "8.10.50", @@ -566,6 +581,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "optional": true, "requires": { "event-target-shim": "^5.0.0" } @@ -911,7 +927,8 @@ "bignumber.js": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", - "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==", + "optional": true }, "bl": { "version": "1.2.2", @@ -1788,6 +1805,7 @@ "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "optional": true, "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -2345,7 +2363,8 @@ "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "optional": true }, "events": { "version": "1.1.1", @@ -2477,7 +2496,8 @@ "fast-text-encoding": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", - "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==" + "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==", + "optional": true }, "faye-websocket": { "version": "0.11.3", @@ -2729,6 +2749,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.0.1.tgz", "integrity": "sha512-c1NXovTxkgRJTIgB2FrFmOFg4YIV6N/bAa4f/FZ4jIw13Ql9ya/82x69CswvotJhbV3DiGnlTZwoq2NVXk2Irg==", + "optional": true, "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -2739,7 +2760,8 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "optional": true } } }, @@ -2794,6 +2816,7 @@ "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", + "optional": true, "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -2805,6 +2828,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "optional": true, "requires": { "event-target-shim": "^5.0.0" } @@ -2879,7 +2903,8 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "optional": true }, "node-forge": { "version": "0.8.5", @@ -2890,7 +2915,8 @@ "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "optional": true }, "yallist": { "version": "3.0.3", @@ -3677,6 +3703,7 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "optional": true, "requires": { "bignumber.js": "^7.0.0" } @@ -3867,9 +3894,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.13.tgz", + "integrity": "sha512-vm3/XWXfWtRua0FkUyEHBZy8kCPjErNBT9fJx8Zvs+U6zjqPbTUOpkaoum3O5uiA8sm+yNMHXfYkTUHFoMxFNA==" }, "lodash.at": { "version": "4.6.0", @@ -3952,7 +3979,8 @@ "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "optional": true }, "lowercase-keys": { "version": "1.0.1", @@ -4751,6 +4779,7 @@ "version": "6.8.8", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", + "optional": true, "requires": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", @@ -4770,7 +4799,8 @@ "@types/node": { "version": "10.14.12", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", - "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==" + "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==", + "optional": true } } }, @@ -4801,6 +4831,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "optional": true, "requires": { "duplexify": "^3.6.0", "inherits": "^2.0.3", @@ -4811,6 +4842,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "optional": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -5686,6 +5718,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "optional": true, "requires": { "stubs": "^3.0.0" } @@ -5693,7 +5726,8 @@ "stream-shift": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", - "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" + "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=", + "optional": true }, "stream-to-array": { "version": "2.3.0", @@ -5802,7 +5836,8 @@ "stubs": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=" + "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=", + "optional": true }, "superagent": { "version": "3.8.3", @@ -5922,6 +5957,7 @@ "version": "3.11.3", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", + "optional": true, "requires": { "https-proxy-agent": "^2.2.1", "node-fetch": "^2.2.0", @@ -5931,7 +5967,8 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "optional": true } } }, @@ -5958,6 +5995,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "optional": true, "requires": { "readable-stream": "2 || 3" } diff --git a/package.json b/package.json index 92b27bc..6117bc4 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "firebase-admin": "^8.2.0", "helmet": "^3.15.1", "json-cycle": "^1.3.0", - "lodash": "^4.17.11", + "lodash": "^4.17.13", "moment": "^2.24.0", "morgan": "^1.9.1", "nanoid": "^2.0.3", From 145062845d0b82d7f2db8f1dfeeeb8bf18a9795b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2019 01:53:44 +0000 Subject: [PATCH 34/52] Bump axios from 0.18.0 to 0.18.1 Bumps [axios](https://github.com/axios/axios) from 0.18.0 to 0.18.1. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v0.18.1/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v0.18.0...v0.18.1) Signed-off-by: dependabot[bot] --- package-lock.json | 38 +++++++++++++++++++++++++++++++++----- package.json | 2 +- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 11260b5..3c39e9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -874,12 +874,40 @@ } }, "axios": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", - "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", + "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", "requires": { - "follow-redirects": "^1.3.0", - "is-buffer": "^1.1.5" + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "follow-redirects": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "requires": { + "debug": "=3.1.0" + } + }, + "is-buffer": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==" + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, "balanced-match": { diff --git a/package.json b/package.json index 6117bc4..e24c72e 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "@sentry/node": "^4.6.4", "aws-param-store": "^2.1.0", "aws-sdk": "^2.399.0", - "axios": "^0.18.0", + "axios": "^0.18.1", "body-parser": "^1.18.3", "contentful-management": "^5.7.0", "dynamodb": "^1.2.0", From d8d0694998c441ac962781ea8c45ff994613d00d Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Thu, 29 Aug 2019 21:40:23 -0400 Subject: [PATCH 35/52] Add limited no-creds Discourse API for fetching like/post counts --- index.js | 27 +++++++++++++++++++++++++++ serverless.yml | 1 + src/creds.js | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/index.js b/index.js index 1b4615f..b1e8d0d 100644 --- a/index.js +++ b/index.js @@ -770,6 +770,33 @@ async function init() { ), ); + app.get( + '*/discourse/counts/:topicId', + wrapAsync( + async (req, res) => { + const { baseUrl, token } = await getCreds('discourse'); + const { topicId } = req.params; + const url = `${baseUrl}/t/${topicId}.json`; + try { + const discourseRes = await axios.get(url, { + headers: { + 'api-key': token, + 'api-username': 'system', + }, + }); + const fields = ['like_count', 'posts_count']; + res.json(_.pick(discourseRes.data, fields)); + } catch (err) { + // TODO: handle errors better? + // don't report to Sentry, though; it's a client-side error, + // so if the client is the app, we'll report the error there + console.error(err); + res.status(err.response.status).end() + } + }, + ), + ) + app.use( // eslint-disable-next-line async (error, req, res, next) => { diff --git a/serverless.yml b/serverless.yml index 32033d4..35004ca 100644 --- a/serverless.yml +++ b/serverless.yml @@ -68,6 +68,7 @@ functions: - http: GET /contentful/{proxy+} - http: GET /rss/{proxy+} - http: POST /contentful-webhook + - http: GET /discourse/counts/{proxy+} sync: handler: src/sync.handler timeout: 300 diff --git a/src/creds.js b/src/creds.js index 63f00e0..2ad6044 100644 --- a/src/creds.js +++ b/src/creds.js @@ -27,6 +27,10 @@ const credSpecs = { firebase: { serviceAccount: `/${stage}/FIREBASE_SERVICE_ACCOUNT`, }, + discourse: { + baseUrl: `/${stage}/DISCOURSE_BASE_URL`, + token: `/${stage}/DISCOURSE_API_TOKEN`, + }, }; module.exports.getCreds = async (name) => { From 5d4bf8a306fda612c6422f3f046c2f42aa04f2c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2019 12:54:39 +0000 Subject: [PATCH 36/52] Bump eslint-utils from 1.3.1 to 1.4.2 Bumps [eslint-utils](https://github.com/mysticatea/eslint-utils) from 1.3.1 to 1.4.2. - [Release notes](https://github.com/mysticatea/eslint-utils/releases) - [Commits](https://github.com/mysticatea/eslint-utils/compare/v1.3.1...v1.4.2) Signed-off-by: dependabot[bot] --- package-lock.json | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3c39e9b..06425a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2326,10 +2326,13 @@ } }, "eslint-utils": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.3.1.tgz", - "integrity": "sha512-Z7YjnIldX+2XMcjr7ZkgEsOj/bREONV60qYeB/bjMAqqqZ4zxKyWX+BOUkdmRmA9riiIPVvo5x86m5elviOk0Q==", - "dev": true + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.2.tgz", + "integrity": "sha512-eAZS2sEUMlIeCjBeubdj45dmBHQwPHWyBcT1VSYB7o9x9WRRqKxyUoiXlRjyAwzN7YEzHJlYg0NmzDRWx6GP4Q==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.0.0" + } }, "eslint-visitor-keys": { "version": "1.0.0", From ee83183a456da621c991ede01d8499b12fe2e61f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2019 12:56:24 +0000 Subject: [PATCH 37/52] Bump mixin-deep from 1.3.1 to 1.3.2 Bumps [mixin-deep](https://github.com/jonschlinkert/mixin-deep) from 1.3.1 to 1.3.2. - [Release notes](https://github.com/jonschlinkert/mixin-deep/releases) - [Commits](https://github.com/jonschlinkert/mixin-deep/compare/1.3.1...1.3.2) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 06425a0..6b5d2f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4143,9 +4143,9 @@ "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", "requires": { "for-in": "^1.0.2", "is-extendable": "^1.0.1" From 8a2a5504278507fda9596b037e3e7c5c4570f0a6 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Mon, 3 Feb 2020 22:41:17 -0500 Subject: [PATCH 38/52] Fix security warnings from outdated node packages --- package-lock.json | 327 +++++++++++++++++++++++++--------------------- package.json | 4 +- 2 files changed, 179 insertions(+), 152 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6b5d2f1..004ff57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -457,74 +457,108 @@ "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=", "optional": true }, + "@sentry/apm": { + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/apm/-/apm-5.12.0.tgz", + "integrity": "sha512-tpD6c0dVKkG+QPEubTTr54TDFdR/FWldnHwkhq4CST2uwkEm26o95iuNhMx9WE7EJhpXjaFKpZB0TB8frFIqkg==", + "requires": { + "@sentry/browser": "5.12.0", + "@sentry/hub": "5.12.0", + "@sentry/minimal": "5.12.0", + "@sentry/types": "5.12.0", + "@sentry/utils": "5.12.0", + "tslib": "^1.9.3" + } + }, + "@sentry/browser": { + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-5.12.0.tgz", + "integrity": "sha512-e8uQML/1Wz2A6610yEvTdICf7L2IH15z6kcjwEqTsaD5uBCmpCiebGZABb45OSe9u8J0xccqi5G7M8lcxj1L7w==", + "requires": { + "@sentry/core": "5.12.0", + "@sentry/types": "5.12.0", + "@sentry/utils": "5.12.0", + "tslib": "^1.9.3" + } + }, "@sentry/core": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-4.6.4.tgz", - "integrity": "sha512-NGl2nkAaQ8dGqJAMS1Hb+7RyVjW4tmCbK6d7H/zKnOpBuU+qSW4XCm2NoGLLa8qb4SZUPIBRv6U0ByvEQlGtqw==", - "requires": { - "@sentry/hub": "4.6.4", - "@sentry/minimal": "4.6.4", - "@sentry/types": "4.5.3", - "@sentry/utils": "4.6.4", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-5.12.0.tgz", + "integrity": "sha512-wY4rsoX71QsGpcs9tF+OxKgDPKzIFMRvFiSRcJoPMfhFsTilQ/CBMn/c3bDtWQd9Bnr/ReQIL6NbnIjUsPHA4Q==", + "requires": { + "@sentry/hub": "5.12.0", + "@sentry/minimal": "5.12.0", + "@sentry/types": "5.12.0", + "@sentry/utils": "5.12.0", "tslib": "^1.9.3" } }, "@sentry/hub": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-4.6.4.tgz", - "integrity": "sha512-R3ACxUZbrAMP6vyIvt1k4bE3OIyg1CzbEhzknKljPrk1abVmJVP7W/X1vBysdRtI3m/9RjOSO7Lxx3XXqoHoQg==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-5.12.0.tgz", + "integrity": "sha512-3k7yE8BEVJsKx8mR4LcI4IN0O8pngmq44OcJ/fRUUBAPqsT38jsJdP2CaWhdlM1jiNUzUDB1ktBv6/lY+VgcoQ==", "requires": { - "@sentry/types": "4.5.3", - "@sentry/utils": "4.6.4", + "@sentry/types": "5.12.0", + "@sentry/utils": "5.12.0", "tslib": "^1.9.3" } }, "@sentry/minimal": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-4.6.4.tgz", - "integrity": "sha512-jZa9mfzDzJI98tg6uxFG3gdVLyz0nOHpLP9H8Kn/BelZ7WEG/ogB8PDi1hI9JvCTXAr8kV81mEecldADa9L9Yg==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-5.12.0.tgz", + "integrity": "sha512-fk73meyz4k4jCg9yzbma+WkggsfEIQWI2e2TWfYsRGcrV3RnlSrXyM4D91/A8Bjx10SNezHPUFHjasjlHXOkyA==", "requires": { - "@sentry/hub": "4.6.4", - "@sentry/types": "4.5.3", + "@sentry/hub": "5.12.0", + "@sentry/types": "5.12.0", "tslib": "^1.9.3" } }, "@sentry/node": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-4.6.4.tgz", - "integrity": "sha512-nfaLB+cE0dddjWD0yI0nB/UqXkPw/6FKDRpB1NZ61amAM4QRXa4hRTdHvqjUovzV/5/pVMQYsOyCk0pNWMtMUQ==", - "requires": { - "@sentry/core": "4.6.4", - "@sentry/hub": "4.6.4", - "@sentry/types": "4.5.3", - "@sentry/utils": "4.6.4", - "@types/stack-trace": "0.0.29", - "cookie": "0.3.1", - "https-proxy-agent": "2.2.1", - "lru_map": "0.3.3", - "lsmod": "1.0.0", - "stack-trace": "0.0.10", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-5.12.0.tgz", + "integrity": "sha512-Q059wrvv7JDTfdlnI4GCAOW9BLeRxxx8P+TYK/t41ZgEbwTob/VU+rn+p7zQm8beHMM3IBtUKE54AmpHYnhJ/A==", + "requires": { + "@sentry/apm": "5.12.0", + "@sentry/core": "5.12.0", + "@sentry/hub": "5.12.0", + "@sentry/types": "5.12.0", + "@sentry/utils": "5.12.0", + "cookie": "^0.3.1", + "https-proxy-agent": "^4.0.0", + "lru_map": "^0.3.3", "tslib": "^1.9.3" }, "dependencies": { - "stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "requires": { + "agent-base": "5", + "debug": "4" + } } } }, "@sentry/types": { - "version": "4.5.3", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-4.5.3.tgz", - "integrity": "sha512-7ll1PAFNjrBNX9rzy3P2qAQrpQwHaDO3uKj735qsnGw34OtAS8Xr8WYrjI14f9fMPa/XIeWvMPb4GMic28V/ag==" + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-5.12.0.tgz", + "integrity": "sha512-aZbBouBLrKB8wXlztriIagZNmsB+wegk1Jkl6eprqRW/w24Sl/47tiwH8c5S4jYTxdAiJk+SAR10AAuYmIN3zg==" }, "@sentry/utils": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-4.6.4.tgz", - "integrity": "sha512-Tc5R46z7ve9Z+uU34ceDoEUR7skfQgXVIZqjbrTQphgm6EcMSNdRfkK3SJYZL5MNKiKhb7Tt/O3aPBy5bTZy6w==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-5.12.0.tgz", + "integrity": "sha512-fYUadGLbfTCbs4OG5hKCOtv2jrNE4/8LHNABy9DwNJ/t5DVtGqWAZBnxsC+FG6a3nVqCpxjFI9AHlYsJ2wsf7Q==", "requires": { - "@sentry/types": "4.5.3", + "@sentry/types": "5.12.0", "tslib": "^1.9.3" } }, @@ -566,11 +600,6 @@ "@types/tough-cookie": "*" } }, - "@types/stack-trace": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@types/stack-trace/-/stack-trace-0.0.29.tgz", - "integrity": "sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g==" - }, "@types/tough-cookie": { "version": "2.3.5", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", @@ -608,12 +637,9 @@ "dev": true }, "agent-base": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", - "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", - "requires": { - "es6-promisify": "^5.0.0" - } + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" }, "ajv": { "version": "6.8.1", @@ -1017,6 +1043,11 @@ } } }, + "bowser": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.9.0.tgz", + "integrity": "sha512-2ld76tuLBNFekRgmJfT2+3j5MIrP6bFict8WAIT3beq+srz1gcKNAdNKMqHqauQt63NmAa88HfP1/Ypa9Er3HA==" + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -1503,9 +1534,9 @@ "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" }, "content-security-policy-builder": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/content-security-policy-builder/-/content-security-policy-builder-2.0.0.tgz", - "integrity": "sha512-j+Nhmj1yfZAikJLImCvPJFE29x/UuBi+/MWqggGGc515JKaZrjuei2RhULJmy0MsstW3E3htl002bwmBNMKr7w==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-security-policy-builder/-/content-security-policy-builder-2.1.0.tgz", + "integrity": "sha512-/MtLWhJVvJNkA9dVLAp6fg9LxD2gfI6R2Fi1hPmfjYXSahJJzcfvoeDOxSyp4NvxMuwWv3WMssE9o31DoULHrQ==" }, "content-type": { "version": "1.0.4", @@ -1770,9 +1801,9 @@ } }, "dns-prefetch-control": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/dns-prefetch-control/-/dns-prefetch-control-0.1.0.tgz", - "integrity": "sha1-YN20V3dOF48flBXwyrsOhbCzALI=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/dns-prefetch-control/-/dns-prefetch-control-0.2.0.tgz", + "integrity": "sha512-hvSnros73+qyZXhHFjx2CMLwoj3Fe7eR9EJsFsqmcI1bB2OBWL/+0YzaEaKssCHnj/6crawNnUyw74Gm2EKe+Q==" }, "doctrine": { "version": "2.1.0", @@ -1789,9 +1820,9 @@ "integrity": "sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q==" }, "dont-sniff-mimetype": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.0.0.tgz", - "integrity": "sha1-WTKJDcn04vGeXrAqIAJuXl78j1g=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.1.0.tgz", + "integrity": "sha512-ZjI4zqTaxveH2/tTlzS1wFp+7ncxNZaIEWYg3lzZRHkKf5zPT/MnEG6WL0BhHMJUabkh8GeU5NL5j+rEUCb7Ug==" }, "dot-prop": { "version": "4.2.0", @@ -2422,9 +2453,9 @@ "integrity": "sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g=" }, "expect-ct": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.1.1.tgz", - "integrity": "sha512-ngXzTfoRGG7fYens3/RMb6yYoVLvLMfmsSllP/mZPxNHgFq41TmPSLF/nLY7fwoclI2vElvAmILFWGUYqdjfCg==" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.2.0.tgz", + "integrity": "sha512-6SK3MG/Bbhm8MsgyJAylg+ucIOU71/FzyFalcfu5nY19dH8y/z0tBJU0wrNBXD4B27EoQtqPF/9wqH0iYAd04g==" }, "express": { "version": "4.16.4", @@ -2547,9 +2578,9 @@ } }, "feature-policy": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/feature-policy/-/feature-policy-0.2.0.tgz", - "integrity": "sha512-2hGrlv6efG4hscYVZeaYjpzpT6I2OZgYqE2yDUzeAcKj2D1SH0AsEzqJNXzdoglEddcIXQQYop3lD97XpG75Jw==" + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/feature-policy/-/feature-policy-0.3.0.tgz", + "integrity": "sha512-ZtijOTFN7TzCujt1fnNhfWPFPSHeZkesff9AXZj+UEjYBynWNUIYpC87Ve4wHzyexQsImicLu7WsC2LHq7/xrQ==" }, "figures": { "version": "1.7.0", @@ -2722,9 +2753,9 @@ } }, "frameguard": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/frameguard/-/frameguard-3.0.0.tgz", - "integrity": "sha1-e8rUae57lukdEs6zlZx4I1qScuk=" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/frameguard/-/frameguard-3.1.0.tgz", + "integrity": "sha512-TxgSKM+7LTA6sidjOiSZK9wxY0ffMPY3Wta//MqwmX0nZuEHc8QrkV8Fh3ZhMJeiH+Uyh/tcaarImRy8u77O7g==" }, "fresh": { "version": "0.5.2", @@ -3260,25 +3291,25 @@ } }, "helmet": { - "version": "3.15.1", - "resolved": "https://registry.npmjs.org/helmet/-/helmet-3.15.1.tgz", - "integrity": "sha512-hgoNe/sjKlKNvJ3g9Gz149H14BjMMWOCmW/DTXl7IfyKGtIK37GePwZrHNfr4aPXdKVyXcTj26RgRFbPKDy9lw==", + "version": "3.21.2", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-3.21.2.tgz", + "integrity": "sha512-okUo+MeWgg00cKB8Csblu8EXgcIoDyb5ZS/3u0W4spCimeVuCUvVZ6Vj3O2VJ1Sxpyb8jCDvzu0L1KKT11pkIg==", "requires": { "depd": "2.0.0", - "dns-prefetch-control": "0.1.0", - "dont-sniff-mimetype": "1.0.0", - "expect-ct": "0.1.1", - "feature-policy": "0.2.0", - "frameguard": "3.0.0", - "helmet-crossdomain": "0.3.0", - "helmet-csp": "2.7.1", - "hide-powered-by": "1.0.0", + "dns-prefetch-control": "0.2.0", + "dont-sniff-mimetype": "1.1.0", + "expect-ct": "0.2.0", + "feature-policy": "0.3.0", + "frameguard": "3.1.0", + "helmet-crossdomain": "0.4.0", + "helmet-csp": "2.9.4", + "hide-powered-by": "1.1.0", "hpkp": "2.0.0", - "hsts": "2.1.0", - "ienoopen": "1.0.0", - "nocache": "2.0.0", - "referrer-policy": "1.1.0", - "x-xss-protection": "1.1.0" + "hsts": "2.2.0", + "ienoopen": "1.1.0", + "nocache": "2.1.0", + "referrer-policy": "1.2.0", + "x-xss-protection": "1.3.0" }, "dependencies": { "depd": { @@ -3289,25 +3320,25 @@ } }, "helmet-crossdomain": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/helmet-crossdomain/-/helmet-crossdomain-0.3.0.tgz", - "integrity": "sha512-YiXhj0E35nC4Na5EPE4mTfoXMf9JTGpN4OtB4aLqShKuH9d2HNaJX5MQoglO6STVka0uMsHyG5lCut5Kzsy7Lg==" + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/helmet-crossdomain/-/helmet-crossdomain-0.4.0.tgz", + "integrity": "sha512-AB4DTykRw3HCOxovD1nPR16hllrVImeFp5VBV9/twj66lJ2nU75DP8FPL0/Jp4jj79JhTfG+pFI2MD02kWJ+fA==" }, "helmet-csp": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/helmet-csp/-/helmet-csp-2.7.1.tgz", - "integrity": "sha512-sCHwywg4daQ2mY0YYwXSZRsgcCeerUwxMwNixGA7aMLkVmPTYBl7gJoZDHOZyXkqPrtuDT3s2B1A+RLI7WxSdQ==", + "version": "2.9.4", + "resolved": "https://registry.npmjs.org/helmet-csp/-/helmet-csp-2.9.4.tgz", + "integrity": "sha512-qUgGx8+yk7Xl8XFEGI4MFu1oNmulxhQVTlV8HP8tV3tpfslCs30OZz/9uQqsWPvDISiu/NwrrCowsZBhFADYqg==", "requires": { + "bowser": "^2.7.0", "camelize": "1.0.0", - "content-security-policy-builder": "2.0.0", - "dasherize": "2.0.0", - "platform": "1.3.5" + "content-security-policy-builder": "2.1.0", + "dasherize": "2.0.0" } }, "hide-powered-by": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hide-powered-by/-/hide-powered-by-1.0.0.tgz", - "integrity": "sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/hide-powered-by/-/hide-powered-by-1.1.0.tgz", + "integrity": "sha512-Io1zA2yOA1YJslkr+AJlWSf2yWFkKjvkcL9Ni1XSUqnGLr/qRQe2UI3Cn/J9MsJht7yEVCe0SscY1HgVMujbgg==" }, "hosted-git-info": { "version": "2.7.1", @@ -3321,9 +3352,19 @@ "integrity": "sha1-EOFCJk52IVpdMMROxD3mTe5tFnI=" }, "hsts": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/hsts/-/hsts-2.1.0.tgz", - "integrity": "sha512-zXhh/DqgrTXJ7erTN6Fh5k/xjMhDGXCqdYN3wvxUvGUQvnxcFfUd8E+6vLg/nk3ss1TYMb+DhRl25fYABioTvA==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hsts/-/hsts-2.2.0.tgz", + "integrity": "sha512-ToaTnQ2TbJkochoVcdXYm4HOCliNozlviNsg+X2XQLQvZNI/kCHR9rZxVYpJB3UPcHz80PgxRyWQ7PdU1r+VBQ==", + "requires": { + "depd": "2.0.0" + }, + "dependencies": { + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + } + } }, "http-errors": { "version": "1.6.3", @@ -3342,12 +3383,22 @@ "integrity": "sha1-ksnBN0w1CF912zWexWzCV8u5P6Q=" }, "https-proxy-agent": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", - "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", + "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", "requires": { - "agent-base": "^4.1.0", + "agent-base": "^4.3.0", "debug": "^3.1.0" + }, + "dependencies": { + "agent-base": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", + "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", + "requires": { + "es6-promisify": "^5.0.0" + } + } } }, "iconv-lite": { @@ -3364,9 +3415,9 @@ "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "ienoopen": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/ienoopen/-/ienoopen-1.0.0.tgz", - "integrity": "sha1-NGpCj0dKrI9QzzeE6i0PFvYr2ms=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ienoopen/-/ienoopen-1.1.0.tgz", + "integrity": "sha512-MFs36e/ca6ohEKtinTJ5VvAJ6oDRAYFdYXweUnGY9L9vcoqFOU4n2ZhmJ0C4z/cwGZ3YIQRSB3XZ1+ghZkY5NQ==" }, "ignore": { "version": "4.0.6", @@ -4300,9 +4351,9 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "nocache": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.0.0.tgz", - "integrity": "sha1-ICtIAhoMTL3i34DeFaF0Q8i0OYA=" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.1.0.tgz", + "integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==" }, "node-fetch": { "version": "1.7.3", @@ -4770,11 +4821,6 @@ "find-up": "^2.1.0" } }, - "platform": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.5.tgz", - "integrity": "sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==" - }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -4993,9 +5039,9 @@ } }, "referrer-policy": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.1.0.tgz", - "integrity": "sha1-NXdOtzW/UPtsB46DM0tHI1AgfXk=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz", + "integrity": "sha512-LgQJIuS6nAy1Jd88DCQRemyE3mS+ispwlqMk3b0yjZ257fI1v9c+/p6SD5gP5FGyXUIgrNOAfmyioHwZtYv2VA==" }, "regex-not": { "version": "1.0.2", @@ -5400,9 +5446,9 @@ "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" }, "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", "requires": { "extend-shallow": "^2.0.1", "is-extendable": "^0.1.1", @@ -6132,39 +6178,20 @@ } }, "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", "is-extendable": "^0.1.1", - "set-value": "^0.4.3" + "set-value": "^2.0.1" }, "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - }, "is-extendable": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=" - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } } } }, @@ -6445,9 +6472,9 @@ } }, "x-xss-protection": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/x-xss-protection/-/x-xss-protection-1.1.0.tgz", - "integrity": "sha512-rx3GzJlgEeZ08MIcDsU2vY2B1QEriUKJTSiNHHUIem6eg9pzVOr2TL3Y4Pd6TMAM5D5azGjcxqI62piITBDHVg==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/x-xss-protection/-/x-xss-protection-1.3.0.tgz", + "integrity": "sha512-kpyBI9TlVipZO4diReZMAHWtS0MMa/7Kgx8hwG/EuZLiA6sg4Ah/4TRdASHhRRN3boobzcYgFRUFSgHRge6Qhg==" }, "xdg-basedir": { "version": "3.0.0", diff --git a/package.json b/package.json index e24c72e..1a9ccb4 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@hapi/joi": "^15.0.3", - "@sentry/node": "^4.6.4", + "@sentry/node": "^5.12.0", "aws-param-store": "^2.1.0", "aws-sdk": "^2.399.0", "axios": "^0.18.1", @@ -20,7 +20,7 @@ "dynamodb": "^1.2.0", "express": "^4.16.4", "firebase-admin": "^8.2.0", - "helmet": "^3.15.1", + "helmet": "^3.21.2", "json-cycle": "^1.3.0", "lodash": "^4.17.13", "moment": "^2.24.0", From 107589db55420daf57d58739ba48e26fcef50b68 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 5 Feb 2020 22:07:47 -0500 Subject: [PATCH 39/52] Bump node version --- serverless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serverless.yml b/serverless.yml index 35004ca..e714b61 100644 --- a/serverless.yml +++ b/serverless.yml @@ -2,7 +2,7 @@ service: ${self:custom.service}-${self:custom.namespace} provider: name: aws - runtime: nodejs8.10 + runtime: nodejs12.x stage: dev region: us-east-1 tracing: From 60a299fde32be9e1f12cf0f2e329297857f3d344 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 8 Feb 2020 22:53:32 -0500 Subject: [PATCH 40/52] Switch to non-log-spamming fork of patreonjs --- index.js | 2 +- package-lock.json | 32 ++++++++++++++++---------------- package.json | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/index.js b/index.js index b1e8d0d..d2a8283 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ const helmet = require('helmet'); const morgan = require('morgan'); const _ = require('lodash'); const Sentry = require('@sentry/node'); -const { patreon: patreonAPI } = require('patreon'); +const { patreon: patreonAPI } = require('@theliturgists/patreon'); const RSS = require('rss'); const striptags = require('striptags'); const generate = require('nanoid/generate'); diff --git a/package-lock.json b/package-lock.json index 004ff57..15012d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -562,6 +562,22 @@ "tslib": "^1.9.3" } }, + "@theliturgists/jsonapi-datastore": { + "version": "0.4.0-beta-3", + "resolved": "https://registry.npmjs.org/@theliturgists/jsonapi-datastore/-/jsonapi-datastore-0.4.0-beta-3.tgz", + "integrity": "sha512-E/JQ7W/sYsV+XDWYlXxAizxUedILYOMdfdEY/fyMsC1rvs1TjOuDoxemkQNjbg+2tq/gFx7fihvJZ9MU6Ca7mw==" + }, + "@theliturgists/patreon": { + "version": "0.4.1-2", + "resolved": "https://registry.npmjs.org/@theliturgists/patreon/-/patreon-0.4.1-2.tgz", + "integrity": "sha512-1h14owquZffirYjqH61JSWtAMuLXyALBk5fQoTha5KL/oEFKNYAzHyAZfCVOG9ZgvIrR5eOK9cySGAz7TlBv6g==", + "requires": { + "@theliturgists/jsonapi-datastore": "^0.4.0-beta-3", + "form-urlencoded": "^2.0.4", + "is-plain-object": "^2.0.4", + "isomorphic-fetch": "^2.2.1" + } + }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -3833,11 +3849,6 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, - "jsonapi-datastore": { - "version": "0.4.0-beta", - "resolved": "https://registry.npmjs.org/jsonapi-datastore/-/jsonapi-datastore-0.4.0-beta.tgz", - "integrity": "sha1-tJn86STUXivDxheGgVIAY+I2HxA=" - }, "jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", @@ -4778,17 +4789,6 @@ "pify": "^2.0.0" } }, - "patreon": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/patreon/-/patreon-0.4.1.tgz", - "integrity": "sha512-aLhjx4rg2BArTq0Kg61MrM4dkJnTQ9kPN8F6a2IlQoYVEtIH7kUK/dprClTx+QYQKlXMfKksN9NCux1YarQJsQ==", - "requires": { - "form-urlencoded": "^2.0.4", - "is-plain-object": "^2.0.4", - "isomorphic-fetch": "^2.2.1", - "jsonapi-datastore": "^0.4.0-beta" - } - }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", diff --git a/package.json b/package.json index 1a9ccb4..12bcc19 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dependencies": { "@hapi/joi": "^15.0.3", "@sentry/node": "^5.12.0", + "@theliturgists/patreon": "^0.4.1-2", "aws-param-store": "^2.1.0", "aws-sdk": "^2.399.0", "axios": "^0.18.1", @@ -26,7 +27,6 @@ "moment": "^2.24.0", "morgan": "^1.9.1", "nanoid": "^2.0.3", - "patreon": "^0.4.1", "progress": "^2.0.3", "qs": "^6.6.0", "rss": "^1.2.2", From f83bafea930ef7081eeba8dbb1f29d2eb46cb3e4 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 3 Mar 2020 21:51:32 -0500 Subject: [PATCH 41/52] Use patreon-pledge module for access control --- index.js | 33 +++++++++++++-------------------- package-lock.json | 28 ++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/index.js b/index.js index d2a8283..c66eae6 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,7 @@ const morgan = require('morgan'); const _ = require('lodash'); const Sentry = require('@sentry/node'); const { patreon: patreonAPI } = require('@theliturgists/patreon'); +const Pledge = require('@theliturgists/patreon-pledge'); const RSS = require('rss'); const striptags = require('striptags'); const generate = require('nanoid/generate'); @@ -21,22 +22,14 @@ const { notifyNewItem } = require('./src/notify'); const stage = process.env.SLS_STAGE; function canAccessPodcast(pledge, podcast) { - return ( - _.get(podcast.fields, 'minimumPledgeDollars', null) === null - || ( - !!pledge - && podcast.fields.minimumPledgeDollars * 100 <= pledge.amount_cents - ) + const patronsOnly = _.get(podcast.fields, 'patronsOnly', false); + return !patronsOnly || ( + pledge && pledge.canAccessPatronPodcasts() ); } function canAccessPatronMedia(pledge) { - // Patrons with the $5 reward tier or above - // get access to both Meditations and Liturgies - // in addition to patrons-only podcasts. - const minMeditationsPledgeCents = 500; - const amount = _.get(pledge, 'amount_cents', 0); - return amount >= minMeditationsPledgeCents; + return pledge && pledge.canAccessMeditations(); } function canAccess(pledge, item, podcasts) { @@ -51,7 +44,7 @@ function canAccess(pledge, item, podcasts) { return true; } -async function getPatreonUser(token) { +async function getPatreonUser(token, opts = { raw: false }) { const client = patreonAPI(token); let resp; try { @@ -70,6 +63,9 @@ async function getPatreonUser(token) { } const { store, rawJson } = resp; + if (opts.raw) { + return rawJson; + } return store.find('user', rawJson.data.id); } @@ -79,12 +75,8 @@ async function getPledge(patreon) { return null; } - const user = await getPatreonUser(token); - - return _.find( - user.pledges, - p => (p.reward.campaign.url === campaignUrl), - ); + const user = await getPatreonUser(token, { raw: true }); + return new Pledge(user); } function filterEntry(entry, pledge, podcasts) { @@ -250,6 +242,7 @@ function handleLiturgistsToken() { req.pledge = await getPledge({ token: newPatreonToken, campaignUrl }); } } catch (err) { + console.log('Error retrieving token mapping:', err); if (tokenMappingObj) { console.log( `Invalid token for patreon user ${tokenMapping.patreonUserId}; ` + @@ -497,7 +490,7 @@ async function init() { res.status(200).json(data); } catch (e) { console.error(e); - res.status(e.status).json(e.json); + res.status(e.status || 500).json(e.json || { error: 'Unkown error' }); } }, ), diff --git a/package-lock.json b/package-lock.json index 15012d2..54560ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,15 @@ "js-tokens": "^4.0.0" } }, + "@babel/polyfill": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.8.3.tgz", + "integrity": "sha512-0QEgn2zkCzqGIkSWWAEmvxD7e00Nm9asTtQvi7HdlYvMhjy/J38V/1Y9ode0zEJeIuxAI0uftiAzqc7nVeWUGg==", + "requires": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.2" + } + }, "@contentful/axios": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/@contentful/axios/-/axios-0.18.0.tgz", @@ -578,6 +587,15 @@ "isomorphic-fetch": "^2.2.1" } }, + "@theliturgists/patreon-pledge": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@theliturgists/patreon-pledge/-/patreon-pledge-0.1.1.tgz", + "integrity": "sha512-5U9czyK4/Uq89e9ukMacUUTz7MoiT65ZDJ2w1oWX6vjhqbueT5mkvtC6wzSiueVo6kg/+p+2qS6y23QOi7llsA==", + "requires": { + "@babel/polyfill": "^7.8.3", + "@theliturgists/jsonapi-datastore": "^0.4.0-beta-3" + } + }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -1598,6 +1616,11 @@ "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=" }, + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==" + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -5043,6 +5066,11 @@ "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz", "integrity": "sha512-LgQJIuS6nAy1Jd88DCQRemyE3mS+ispwlqMk3b0yjZ257fI1v9c+/p6SD5gP5FGyXUIgrNOAfmyioHwZtYv2VA==" }, + "regenerator-runtime": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz", + "integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==" + }, "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", diff --git a/package.json b/package.json index 12bcc19..885dc6b 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "@hapi/joi": "^15.0.3", "@sentry/node": "^5.12.0", "@theliturgists/patreon": "^0.4.1-2", + "@theliturgists/patreon-pledge": "^0.1.1", "aws-param-store": "^2.1.0", "aws-sdk": "^2.399.0", "axios": "^0.18.1", From c3579b45d053508d8812e99f6319fde5c1ee9c90 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 4 Mar 2020 22:47:14 -0500 Subject: [PATCH 42/52] Update reference of perms required for deploy and cert verification --- aws/deploy_policy.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/aws/deploy_policy.json b/aws/deploy_policy.json index 2e77b06..52c4552 100644 --- a/aws/deploy_policy.json +++ b/aws/deploy_policy.json @@ -143,7 +143,8 @@ "Action": [ "route53:ChangeResourceRecordSets", "route53:ListHostedZones", - "route53:ListResourceRecordSets" + "route53:ListResourceRecordSets", + "route53:GetHostedZone" ], "Resource": "*" }, @@ -153,20 +154,21 @@ "acm:AddTagsToCertificate", "acm:RequestCertificate", "acm:DescribeCertificate", - "acm:DeleteCertificate" + "acm:DeleteCertificate", + "acm:ListCertificates" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ - "apigateway:GET", - "apigateway:POST", - "apigateway:DELETE" + "apigateway:*" ], "Resource": [ "arn:aws:apigateway:*::/domainnames", - "arn:aws:apigateway:*::/domainnames/*" + "arn:aws:apigateway:*::/domainnames/*", + "arn:aws:apigateway:*::/domainnames/*/*", + "arn:aws:apigateway:*::/domainnames/*/*/*" ] }, { From 9f7146b426526eb1ab7180fd4c8a2486298be661 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 17 Mar 2020 23:02:41 -0400 Subject: [PATCH 43/52] Send scoped notifications --- serverless.yml | 1 + src/notify.js | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/serverless.yml b/serverless.yml index e714b61..c10095d 100644 --- a/serverless.yml +++ b/serverless.yml @@ -8,6 +8,7 @@ provider: tracing: lambda: true environment: + SLS_NAMESPACE: ${self:custom.namespace} SLS_STAGE: ${self:custom.stage} DYNAMODB_TABLE_TOKENS: ${self:custom.dynamodbTables.tokens} iamRoleStatements: diff --git a/src/notify.js b/src/notify.js index 093627a..d4452d2 100644 --- a/src/notify.js +++ b/src/notify.js @@ -20,7 +20,7 @@ const TOPIC_PATRON_PODCAST = 'new-patron-podcast'; const TOPIC_PATRON_MEDITATION = 'new-patron-meditation'; const TOPIC_PATRON_LITURGY = 'new-patron-liturgy'; -function getTopic(entry, collectionEntry) { +function getUnscopedTopic(entry, collectionEntry) { if (_.get(entry, 'fields.isFreePreview.en-US')) { return TOPIC_PUBLIC_MEDIA; } @@ -38,6 +38,20 @@ function getTopic(entry, collectionEntry) { return TOPIC_PUBLIC_MEDIA; } +function getTopic(entry, collectionEntry) { + const topic = getUnscopedTopic(entry, collectionEntry); + const namespace = process.env.SLS_NAMESPACE; + const stage = process.env.SLS_STAGE; + let scope; + if (namespace === stage) { + // avoid unnecessary length of "staging-staging" + scope = stage; + } else { + scope = `${namespace}-${stage}`; + } + return `${topic}-${scope}`; +} + function getImageUrl(collectionEntry) { if (collectionEntry.fields.image) { return `https:${collectionEntry.fields.image.fields.file.url}` From 9f8bd2f922be8b18cbbf8cc080f28e5ab5e53086 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 18 Mar 2020 22:38:47 -0400 Subject: [PATCH 44/52] Put subtitle in first line of description --- src/notify.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/notify.js b/src/notify.js index d4452d2..7b10b5f 100644 --- a/src/notify.js +++ b/src/notify.js @@ -72,20 +72,27 @@ function stripTags(html) { return html.replace(/<[^>]+>/g, ''); } -function formatDescription(description) { - return truncate(stripTags(description)); +function formatSubtitle(collectionEntry) { + // collection is only missing for uncategorized meditations + const title = _.get(collectionEntry, 'fields.title', 'Uncategorized'); + + if (collectionEntry.sys.contentType.sys.id === 'meditationCategory') { + return `Meditation: ${title}`; + } + return title; } function makeNotification(entry, collectionEntry) { const topic = getTopic(entry, collectionEntry); const title = entry.fields.title['en-US']; - const subtitle = collectionEntry.fields.title; + const subtitle = formatSubtitle(collectionEntry); + const description = formatDescription(entry.fields.description['en-US']); return { topic, notification: { - title: `${title} (${subtitle})`, - body: formatDescription(entry.fields.description['en-US']), + title, + body: `${subtitle}\n\n${description}`, }, android: { notification: { From a43f2c9fa9243dbdb198daaedfcc4cbce2add9b4 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Wed, 18 Mar 2020 22:39:33 -0400 Subject: [PATCH 45/52] Use html-to-text to render description --- package-lock.json | 97 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + src/notify.js | 10 ++++- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 54560ed..ceb03a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1853,11 +1853,54 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz", + "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", + "requires": { + "domelementtype": "^2.0.1", + "entities": "^2.0.0" + }, + "dependencies": { + "domelementtype": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.0.1.tgz", + "integrity": "sha512-5HOHUDsYZWV8FGWN0Njbr/Rn7f/eWSQi1v7+HsUVwXgn8nWWlL64zKDkS0n8ZmQ3mlWOMuXOnR+7Nx/5tMO5AQ==" + }, + "entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.0.tgz", + "integrity": "sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw==" + } + } + }, "dom-storage": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/dom-storage/-/dom-storage-2.1.0.tgz", "integrity": "sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q==" }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dont-sniff-mimetype": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.1.0.tgz", @@ -3329,6 +3372,11 @@ } } }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + }, "helmet": { "version": "3.21.2", "resolved": "https://registry.npmjs.org/helmet/-/helmet-3.21.2.tgz", @@ -3405,6 +3453,47 @@ } } }, + "html-tags": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-3.1.0.tgz", + "integrity": "sha512-1qYz89hW3lFDEazhjW0yVAV87lw8lVkrJocr72XmBkMKsoSVJCQx3W8BXsC7hO2qAt8BoVjYjtAcZ9perqGnNg==" + }, + "html-to-text": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-5.1.1.tgz", + "integrity": "sha512-Bci6bD/JIfZSvG4s0gW/9mMKwBRoe/1RWLxUME/d6WUSZCdY7T60bssf/jFf7EYXRyqU4P5xdClVqiYU0/ypdA==", + "requires": { + "he": "^1.2.0", + "htmlparser2": "^3.10.1", + "lodash": "^4.17.11", + "minimist": "^1.2.0" + } + }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "http-errors": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -3650,6 +3739,14 @@ "number-is-nan": "^1.0.0" } }, + "is-html": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-html/-/is-html-2.0.0.tgz", + "integrity": "sha512-S+OpgB5i7wzIue/YSE5hg0e5ZYfG3hhpNh9KGl6ayJ38p7ED6wxQLd1TV91xHpcTvw90KMJ9EwN3F/iNflHBVg==", + "requires": { + "html-tags": "^3.0.0" + } + }, "is-installed-globally": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.1.0.tgz", diff --git a/package.json b/package.json index 885dc6b..4427be0 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,8 @@ "express": "^4.16.4", "firebase-admin": "^8.2.0", "helmet": "^3.21.2", + "html-to-text": "^5.1.1", + "is-html": "^2.0.0", "json-cycle": "^1.3.0", "lodash": "^4.17.13", "moment": "^2.24.0", diff --git a/src/notify.js b/src/notify.js index 7b10b5f..b827f27 100644 --- a/src/notify.js +++ b/src/notify.js @@ -1,5 +1,7 @@ const _ = require('lodash'); const firebase = require('firebase-admin'); +const isHtml = require('is-html'); +const htmlToText = require('html-to-text'); const { getCreds } = require('./creds'); @@ -68,8 +70,12 @@ function truncate(str, limit = 1024) { return str.slice(0, limit - 3) + '...'; } -function stripTags(html) { - return html.replace(/<[^>]+>/g, ''); +function formatDescription(description) { + let text = description; + if (isHtml(description)) { + text = htmlToText.fromString(description); + } + return truncate(text); } function formatSubtitle(collectionEntry) { From b1e25227fd3c51b8f1e126f21dc3ea2bb3711429 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Tue, 24 Mar 2020 23:43:34 -0400 Subject: [PATCH 46/52] Don't sync episodes that were published more than a week ago This avoids overwriting descriptions that have changed in the RSS feed for some reason. (This happened at some point during the switch from Podbean to Megaphone.) --- package.json | 2 +- src/sync.js | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 4427be0..7b08c43 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.0.1", "private": true, "scripts": { - "sync": "node ./src/sync.js", + "sync": "sls invoke -f sync", "lint": "eslint", "deploy": "sls deploy", "auth-deploy": "npm run auth-sls deploy", diff --git a/src/sync.js b/src/sync.js index c6b455d..6634cfd 100644 --- a/src/sync.js +++ b/src/sync.js @@ -225,11 +225,17 @@ async function syncEpisode( }; if (!_.isEqual(existingEpisode.fields, newFields)) { - logProgress(progress, `Updating episode: "${title}"`); - status = 'updated'; - existingEpisode.fields = newFields; - existingEpisode = await existingEpisode.update(); - await existingEpisode.publish(); + const publishedAt = moment(existingEpisode.fields.publishedAt['en-US']); + const oneWeekInMillis = 1000 * 60 * 60 * 24 * 7; + if (moment().diff(publishedAt) > oneWeekInMillis) { + logProgress(progress, `Not updating episode: "${title}" (older than one week)`); + } else { + logProgress(progress, `Updating episode: "${title}"`); + status = 'updated'; + existingEpisode.fields = newFields; + existingEpisode = await existingEpisode.update(); + await existingEpisode.publish(); + } } return { episode: existingEpisode, status }; From 2820fef30d35a4d4c1cfc2f0f4d96623efa485d4 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Thu, 2 Apr 2020 23:07:32 -0400 Subject: [PATCH 47/52] Update patreon-pledge package --- package-lock.json | 20 ++++++++++---------- package.json | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index ceb03a9..a4f9b3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,12 +25,12 @@ } }, "@babel/polyfill": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.8.3.tgz", - "integrity": "sha512-0QEgn2zkCzqGIkSWWAEmvxD7e00Nm9asTtQvi7HdlYvMhjy/J38V/1Y9ode0zEJeIuxAI0uftiAzqc7nVeWUGg==", + "version": "7.8.7", + "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.8.7.tgz", + "integrity": "sha512-LeSfP9bNZH2UOZgcGcZ0PIHUt1ZuHub1L3CVmEyqLxCeDLm4C5Gi8jRH8ZX2PNpDhQCo0z6y/+DIs2JlliXW8w==", "requires": { "core-js": "^2.6.5", - "regenerator-runtime": "^0.13.2" + "regenerator-runtime": "^0.13.4" } }, "@contentful/axios": { @@ -588,9 +588,9 @@ } }, "@theliturgists/patreon-pledge": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@theliturgists/patreon-pledge/-/patreon-pledge-0.1.1.tgz", - "integrity": "sha512-5U9czyK4/Uq89e9ukMacUUTz7MoiT65ZDJ2w1oWX6vjhqbueT5mkvtC6wzSiueVo6kg/+p+2qS6y23QOi7llsA==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@theliturgists/patreon-pledge/-/patreon-pledge-0.2.0.tgz", + "integrity": "sha512-mkj5t7GBG13qmIHuHPuByUbc16gwAJqGpWKODBAJSqnvMtM2SfM/hTDRgyl5CC7Hz7YqvoxyCBihXwLzQARKkA==", "requires": { "@babel/polyfill": "^7.8.3", "@theliturgists/jsonapi-datastore": "^0.4.0-beta-3" @@ -5164,9 +5164,9 @@ "integrity": "sha512-LgQJIuS6nAy1Jd88DCQRemyE3mS+ispwlqMk3b0yjZ257fI1v9c+/p6SD5gP5FGyXUIgrNOAfmyioHwZtYv2VA==" }, "regenerator-runtime": { - "version": "0.13.3", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz", - "integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==" + "version": "0.13.5", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", + "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==" }, "regex-not": { "version": "1.0.2", diff --git a/package.json b/package.json index 7b08c43..79037d0 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "@hapi/joi": "^15.0.3", "@sentry/node": "^5.12.0", "@theliturgists/patreon": "^0.4.1-2", - "@theliturgists/patreon-pledge": "^0.1.1", + "@theliturgists/patreon-pledge": "^0.2.0", "aws-param-store": "^2.1.0", "aws-sdk": "^2.399.0", "axios": "^0.18.1", From 364f9fd77c16e1bc347170f4f53c0b79f2aea8c0 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Thu, 2 Apr 2020 23:19:44 -0400 Subject: [PATCH 48/52] Capture errors during content filtering --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index c66eae6..280057b 100644 --- a/index.js +++ b/index.js @@ -465,6 +465,7 @@ async function init() { try { return await filterData(data, pledge); } catch (e) { + Sentry.captureException(e); e.json = { error: ( 'Error verifying Patreon status. ' From 1b30cd4c3227880a44ce056bb34ef596ba0281d1 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Fri, 3 Apr 2020 13:36:12 -0400 Subject: [PATCH 49/52] Omit podcast episodes without a published podcast --- index.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 280057b..62d8ba7 100644 --- a/index.js +++ b/index.js @@ -80,6 +80,21 @@ async function getPledge(patreon) { } function filterEntry(entry, pledge, podcasts) { + if (entry.sys.contentType.sys.id === 'podcastEpisode') { + const podcast = podcasts[entry.fields.podcast.sys.id]; + if (!podcast) { + // Note: `podcast` can be `null` here if the podcast + // is assigned but isn't published. Such episodes are + // probably published by accident and should not be + // accessible to any user, so we return null here + // so that the caller knows to omit this entry. + const msg = `Podcast episode without a podcast: ${entry.fields.title}`; + console.error(msg); + Sentry.captureMessage(msg); + return null; + } + } + if (canAccess(pledge, entry, podcasts)) { return _.set(entry, 'fields.patronsOnly', false); } @@ -112,7 +127,9 @@ async function filterData(contentfulData, pledge) { const d = { ...contentfulData, - items: contentfulData.items.map(item => filterEntry(item, pledge, podcasts)), + items: contentfulData.items + .map(item => filterEntry(item, pledge, podcasts)) + .filter(entry => !!entry), }; return d; } From 74a83ff40ec319c04f4276e8ba732a38c12383a8 Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 4 Apr 2020 08:53:02 -0400 Subject: [PATCH 50/52] Fix 500 error when fetching an asset --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 62d8ba7..f1d5541 100644 --- a/index.js +++ b/index.js @@ -80,7 +80,8 @@ async function getPledge(patreon) { } function filterEntry(entry, pledge, podcasts) { - if (entry.sys.contentType.sys.id === 'podcastEpisode') { + const contentType = _.get(entry, 'sys.contentType.sys.id'); + if (contentType === 'podcastEpisode') { const podcast = podcasts[entry.fields.podcast.sys.id]; if (!podcast) { // Note: `podcast` can be `null` here if the podcast From db5e1d108ba5127c1d34715c88a4332352f03c3b Mon Sep 17 00:00:00 2001 From: Brett Higgins Date: Sat, 4 Apr 2020 22:10:34 -0400 Subject: [PATCH 51/52] Add environment to sentry config --- index.js | 1 + package-lock.json | 98 +++++++++++++++++++++++------------------------ package.json | 2 +- 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/index.js b/index.js index f1d5541..5d19f0f 100644 --- a/index.js +++ b/index.js @@ -318,6 +318,7 @@ function redactUrl(url) { async function init() { const sentry = await getCreds('sentry'); + sentry.environment = stage; Sentry.init(sentry); Sentry.configureScope((scope) => { diff --git a/package-lock.json b/package-lock.json index a4f9b3d..b4b5cc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -467,71 +467,71 @@ "optional": true }, "@sentry/apm": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/apm/-/apm-5.12.0.tgz", - "integrity": "sha512-tpD6c0dVKkG+QPEubTTr54TDFdR/FWldnHwkhq4CST2uwkEm26o95iuNhMx9WE7EJhpXjaFKpZB0TB8frFIqkg==", - "requires": { - "@sentry/browser": "5.12.0", - "@sentry/hub": "5.12.0", - "@sentry/minimal": "5.12.0", - "@sentry/types": "5.12.0", - "@sentry/utils": "5.12.0", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/apm/-/apm-5.15.4.tgz", + "integrity": "sha512-gcW225Jls1ShyBXMWN6zZyuVJwBOIQ63sI+URI2NSFsdpBpdpZ8yennIm+oMlSfb25Nzt9SId7TRSjPhlSbTZQ==", + "requires": { + "@sentry/browser": "5.15.4", + "@sentry/hub": "5.15.4", + "@sentry/minimal": "5.15.4", + "@sentry/types": "5.15.4", + "@sentry/utils": "5.15.4", "tslib": "^1.9.3" } }, "@sentry/browser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-5.12.0.tgz", - "integrity": "sha512-e8uQML/1Wz2A6610yEvTdICf7L2IH15z6kcjwEqTsaD5uBCmpCiebGZABb45OSe9u8J0xccqi5G7M8lcxj1L7w==", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-5.15.4.tgz", + "integrity": "sha512-l/auT1HtZM3KxjCGQHYO/K51ygnlcuOrM+7Ga8gUUbU9ZXDYw6jRi0+Af9aqXKmdDw1naNxr7OCSy6NBrLWVZw==", "requires": { - "@sentry/core": "5.12.0", - "@sentry/types": "5.12.0", - "@sentry/utils": "5.12.0", + "@sentry/core": "5.15.4", + "@sentry/types": "5.15.4", + "@sentry/utils": "5.15.4", "tslib": "^1.9.3" } }, "@sentry/core": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-5.12.0.tgz", - "integrity": "sha512-wY4rsoX71QsGpcs9tF+OxKgDPKzIFMRvFiSRcJoPMfhFsTilQ/CBMn/c3bDtWQd9Bnr/ReQIL6NbnIjUsPHA4Q==", - "requires": { - "@sentry/hub": "5.12.0", - "@sentry/minimal": "5.12.0", - "@sentry/types": "5.12.0", - "@sentry/utils": "5.12.0", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-5.15.4.tgz", + "integrity": "sha512-9KP4NM4SqfV5NixpvAymC7Nvp36Zj4dU2fowmxiq7OIbzTxGXDhwuN/t0Uh8xiqlkpkQqSECZ1OjSFXrBldetQ==", + "requires": { + "@sentry/hub": "5.15.4", + "@sentry/minimal": "5.15.4", + "@sentry/types": "5.15.4", + "@sentry/utils": "5.15.4", "tslib": "^1.9.3" } }, "@sentry/hub": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-5.12.0.tgz", - "integrity": "sha512-3k7yE8BEVJsKx8mR4LcI4IN0O8pngmq44OcJ/fRUUBAPqsT38jsJdP2CaWhdlM1jiNUzUDB1ktBv6/lY+VgcoQ==", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-5.15.4.tgz", + "integrity": "sha512-1XJ1SVqadkbUT4zLS0TVIVl99si7oHizLmghR8LMFl5wOkGEgehHSoOydQkIAX2C7sJmaF5TZ47ORBHgkqclUg==", "requires": { - "@sentry/types": "5.12.0", - "@sentry/utils": "5.12.0", + "@sentry/types": "5.15.4", + "@sentry/utils": "5.15.4", "tslib": "^1.9.3" } }, "@sentry/minimal": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-5.12.0.tgz", - "integrity": "sha512-fk73meyz4k4jCg9yzbma+WkggsfEIQWI2e2TWfYsRGcrV3RnlSrXyM4D91/A8Bjx10SNezHPUFHjasjlHXOkyA==", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-5.15.4.tgz", + "integrity": "sha512-GL4GZ3drS9ge+wmxkHBAMEwulaE7DMvAEfKQPDAjg2p3MfcCMhAYfuY4jJByAC9rg9OwBGGehz7UmhWMFjE0tw==", "requires": { - "@sentry/hub": "5.12.0", - "@sentry/types": "5.12.0", + "@sentry/hub": "5.15.4", + "@sentry/types": "5.15.4", "tslib": "^1.9.3" } }, "@sentry/node": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-5.12.0.tgz", - "integrity": "sha512-Q059wrvv7JDTfdlnI4GCAOW9BLeRxxx8P+TYK/t41ZgEbwTob/VU+rn+p7zQm8beHMM3IBtUKE54AmpHYnhJ/A==", - "requires": { - "@sentry/apm": "5.12.0", - "@sentry/core": "5.12.0", - "@sentry/hub": "5.12.0", - "@sentry/types": "5.12.0", - "@sentry/utils": "5.12.0", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-5.15.4.tgz", + "integrity": "sha512-OfdhNEvOJZ55ZkCUcVgctjaZkOw7rmLzO5VyDTSgevA4uLsPaTNXSAeK2GSQBXc5J0KdRpNz4sSIyuxOS4Z7Vg==", + "requires": { + "@sentry/apm": "5.15.4", + "@sentry/core": "5.15.4", + "@sentry/hub": "5.15.4", + "@sentry/types": "5.15.4", + "@sentry/utils": "5.15.4", "cookie": "^0.3.1", "https-proxy-agent": "^4.0.0", "lru_map": "^0.3.3", @@ -558,16 +558,16 @@ } }, "@sentry/types": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-5.12.0.tgz", - "integrity": "sha512-aZbBouBLrKB8wXlztriIagZNmsB+wegk1Jkl6eprqRW/w24Sl/47tiwH8c5S4jYTxdAiJk+SAR10AAuYmIN3zg==" + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-5.15.4.tgz", + "integrity": "sha512-quPHPpeAuwID48HLPmqBiyXE3xEiZLZ5D3CEbU3c3YuvvAg8qmfOOTI6z4Z3Eedi7flvYpnx3n7N3dXIEz30Eg==" }, "@sentry/utils": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-5.12.0.tgz", - "integrity": "sha512-fYUadGLbfTCbs4OG5hKCOtv2jrNE4/8LHNABy9DwNJ/t5DVtGqWAZBnxsC+FG6a3nVqCpxjFI9AHlYsJ2wsf7Q==", + "version": "5.15.4", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-5.15.4.tgz", + "integrity": "sha512-lO8SLBjrUDGADl0LOkd55R5oL510d/1SaI08/IBHZCxCUwI4TiYo5EPECq8mrj3XGfgCyq9osw33bymRlIDuSQ==", "requires": { - "@sentry/types": "5.12.0", + "@sentry/types": "5.15.4", "tslib": "^1.9.3" } }, diff --git a/package.json b/package.json index 79037d0..2e01bd1 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@hapi/joi": "^15.0.3", - "@sentry/node": "^5.12.0", + "@sentry/node": "^5.15.4", "@theliturgists/patreon": "^0.4.1-2", "@theliturgists/patreon-pledge": "^0.2.0", "aws-param-store": "^2.1.0", From f072038a6b33b0b403966dc651c43750c070e67c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 7 Jun 2020 03:05:58 +0000 Subject: [PATCH 52/52] Bump websocket-extensions from 0.1.3 to 0.1.4 Bumps [websocket-extensions](https://github.com/faye/websocket-extensions-node) from 0.1.3 to 0.1.4. - [Release notes](https://github.com/faye/websocket-extensions-node/releases) - [Changelog](https://github.com/faye/websocket-extensions-node/blob/master/CHANGELOG.md) - [Commits](https://github.com/faye/websocket-extensions-node/compare/0.1.3...0.1.4) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b4b5cc8..775fbb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6498,9 +6498,9 @@ } }, "websocket-extensions": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==" + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==" }, "whatwg-fetch": { "version": "3.0.0",