diff --git a/control-plane/package-lock.json b/control-plane/package-lock.json index be00d87e..603ec51b 100644 --- a/control-plane/package-lock.json +++ b/control-plane/package-lock.json @@ -43,6 +43,7 @@ "jwks-rsa": "^3.1.0", "langfuse": "^3.31.0", "lodash": "^4.17.21", + "mailparser": "^3.7.2", "node-cache": "^5.1.2", "pem-jwk": "^2.0.0", "pg": "^8.11.2", @@ -67,6 +68,7 @@ "@types/jsonpath": "^0.2.4", "@types/jsonwebtoken": "^9.0.2", "@types/lodash": "^4.17.6", + "@types/mailparser": "^3.4.5", "@types/pg": "^8.10.2", "@types/skmeans": "^0.11.7", "@typescript-eslint/eslint-plugin": "^7.16.0", @@ -10413,6 +10415,19 @@ "@redis/client": "^1.0.0" } }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "selderee": "^0.11.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/@sentry/core": { "version": "8.41.0", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.41.0.tgz", @@ -13003,6 +13018,30 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mailparser": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/mailparser/-/mailparser-3.4.5.tgz", + "integrity": "sha512-EPERBp7fLeFZh7tS2X36MF7jawUx3Y6/0rXciZah3CTYgwLi3e0kpGUJ6FOmUabgzis/U1g+3/JzrVWbWIOGjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "iconv-lite": "^0.6.3" + } + }, + "node_modules/@types/mailparser/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/@types/memcached": { "version": "2.2.10", "resolved": "https://registry.npmjs.org/@types/memcached/-/memcached-2.2.10.tgz", @@ -15009,6 +15048,61 @@ "node": ">=6.0.0" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.1.tgz", + "integrity": "sha512-xWXmuRnN9OMP6ptPd2+H0cCbcYBULa5YDTbMm/2lvkWvNA3O4wcW+GvzooqBuNM8yy6pl3VIAeJTUUWUbfI5Fw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dot-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", @@ -15233,12 +15327,20 @@ "node": ">= 0.8" } }, + "node_modules/encoding-japanese": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.2.0.tgz", + "integrity": "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A==", + "license": "MIT", + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", "license": "BSD-2-Clause", - "peer": true, "engines": { "node": ">=0.12" }, @@ -16770,6 +16872,15 @@ "node": ">= 0.4" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, "node_modules/headers-polyfill": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", @@ -16783,6 +16894,41 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "license": "MIT" }, + "node_modules/html-to-text": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.11.0", + "deepmerge": "^4.3.1", + "dom-serializer": "^2.0.0", + "htmlparser2": "^8.0.2", + "selderee": "^0.11.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -18170,6 +18316,15 @@ "node": ">=10" } }, + "node_modules/leac": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -18194,6 +18349,42 @@ "node": ">= 0.8.0" } }, + "node_modules/libbase64": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz", + "integrity": "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg==", + "license": "MIT" + }, + "node_modules/libmime": { + "version": "5.3.6", + "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.3.6.tgz", + "integrity": "sha512-j9mBC7eiqi6fgBPAGvKCXJKJSIASanYF4EeA4iBzSG0HxQxmXnR3KbyWqTn4CwsKSebqCv2f5XZfAO6sKzgvwA==", + "license": "MIT", + "dependencies": { + "encoding-japanese": "2.2.0", + "iconv-lite": "0.6.3", + "libbase64": "1.3.0", + "libqp": "2.1.1" + } + }, + "node_modules/libmime/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libqp": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.1.1.tgz", + "integrity": "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==", + "license": "MIT" + }, "node_modules/light-my-request": { "version": "5.14.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.14.0.tgz", @@ -18216,6 +18407,15 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "license": "MIT" }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/locate-character": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", @@ -18433,6 +18633,47 @@ "@jridgewell/sourcemap-codec": "^1.5.0" } }, + "node_modules/mailparser": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/mailparser/-/mailparser-3.7.2.tgz", + "integrity": "sha512-iI0p2TCcIodR1qGiRoDBBwboSSff50vQAWytM5JRggLfABa4hHYCf3YVujtuzV454xrOP352VsAPIzviqMTo4Q==", + "license": "MIT", + "dependencies": { + "encoding-japanese": "2.2.0", + "he": "1.2.0", + "html-to-text": "9.0.5", + "iconv-lite": "0.6.3", + "libmime": "5.3.6", + "linkify-it": "5.0.0", + "mailsplit": "5.4.2", + "nodemailer": "6.9.16", + "punycode.js": "2.3.1", + "tlds": "1.255.0" + } + }, + "node_modules/mailparser/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mailsplit": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/mailsplit/-/mailsplit-5.4.2.tgz", + "integrity": "sha512-4cczG/3Iu3pyl8JgQ76dKkisurZTmxMrA4dj/e8d2jKYcFTZ7MxOzg1gTioTDMPuFXwTrVuN/gxhkrO7wLg7qA==", + "license": "(MIT OR EUPL-1.1+)", + "dependencies": { + "libbase64": "1.3.0", + "libmime": "5.3.6", + "libqp": "2.1.1" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -18842,6 +19083,15 @@ "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", "license": "MIT" }, + "node_modules/nodemailer": { + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.16.tgz", + "integrity": "sha512-psAuZdTIRN08HKVd/E8ObdV6NO7NTBY3KsC30F7M4H1OnmLCUNaS56FpYxyb26zWLSyYF9Ozch9KYHhHegsiOQ==", + "license": "MIT-0", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -19195,6 +19445,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parseley": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "license": "MIT", + "dependencies": { + "leac": "^0.6.0", + "peberminta": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -19256,6 +19519,15 @@ "node": ">=8" } }, + "node_modules/peberminta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/pem-jwk": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pem-jwk/-/pem-jwk-2.0.0.tgz", @@ -19808,6 +20080,15 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -20377,6 +20658,18 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", "license": "BSD-3-Clause" }, + "node_modules/selderee": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "license": "MIT", + "dependencies": { + "parseley": "^0.12.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -20977,6 +21270,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tlds": { + "version": "1.255.0", + "resolved": "https://registry.npmjs.org/tlds/-/tlds-1.255.0.tgz", + "integrity": "sha512-tcwMRIioTcF/FcxLev8MJWxCp+GUALRhFEqbDoZrnowmKSGqPrl5pqS+Sut2m8BgJ6S4FExCSSpGffZ0Tks6Aw==", + "license": "MIT", + "bin": { + "tlds": "bin.js" + } + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -21666,6 +21968,12 @@ "node": ">=14.17" } }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, "node_modules/ulid": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.3.0.tgz", diff --git a/control-plane/package.json b/control-plane/package.json index d48c2750..a66213e8 100644 --- a/control-plane/package.json +++ b/control-plane/package.json @@ -55,6 +55,7 @@ "jwks-rsa": "^3.1.0", "langfuse": "^3.31.0", "lodash": "^4.17.21", + "mailparser": "^3.7.2", "node-cache": "^5.1.2", "pem-jwk": "^2.0.0", "pg": "^8.11.2", @@ -79,6 +80,7 @@ "@types/jsonpath": "^0.2.4", "@types/jsonwebtoken": "^9.0.2", "@types/lodash": "^4.17.6", + "@types/mailparser": "^3.4.5", "@types/pg": "^8.10.2", "@types/skmeans": "^0.11.7", "@typescript-eslint/eslint-plugin": "^7.16.0", @@ -90,4 +92,4 @@ "ts-node": "^10.9.2", "tsx": "^4.7.0" } -} \ No newline at end of file +} diff --git a/control-plane/scripts/sendTestEmail.sh b/control-plane/scripts/sendTestEmail.sh deleted file mode 100755 index 104409b4..00000000 --- a/control-plane/scripts/sendTestEmail.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -MESSAGE_BODY=$(cat <\"},{\"name\":\"Received\",\"value\":\"from mr85p00im-ztdg06011901.me.com (mr85p00im-ztdg06011901.me.com [17.58.23.198]) by inbound-smtp.us-east-1.amazonaws.com with SMTP id io5akr05ppl6s5095kjm6p5rusbbsaode3gn6j81 for test@run.inferable.ai; Tue, 31 Dec 2024 04:15:55 +0000 (UTC)\"},{\"name\":\"X-SES-Spam-Verdict\",\"value\":\"PASS\"},{\"name\":\"X-SES-Virus-Verdict\",\"value\":\"PASS\"},{\"name\":\"Received-SPF\",\"value\":\"pass (spfCheck: domain of icloud.com designates 17.58.23.198 as permitted sender) client-ip=17.58.23.198; envelope-from=john@johnjcsmith.com; helo=mr85p00im-ztdg06011901.me.com;\"},{\"name\":\"Authentication-Results\",\"value\":\"amazonses.com; spf=pass (spfCheck: domain of icloud.com designates 17.58.23.198 as permitted sender) client-ip=17.58.23.198; envelope-from=john@johnjcsmith.com; helo=mr85p00im-ztdg06011901.me.com; dkim=fail header.i=@johnjcsmith.com; dmarc=none header.from=johnjcsmith.com;\"},{\"name\":\"X-SES-RECEIPT\",\"value\":\"AEFBQUFBQUFBQUFHWTBhd3Fsa3ZBdE9GNnJUbWw0ZDFEemdKSlRuNUxNMWkveGFwMGtiVTBzanVNdWE5eEhodFhHM1MrWVhkYnVSN2FDVFNGQ0NIdlQ4V1pjdDVOQVVQL2dPMU9nUHQ0a1dQbEdRdUJqZE94NWt3Ri9XTEtXWlllYkhZVW9TTkZsUHZITUcxZ0VPb255OU9WT3pWS0dsUkZ5L0I4V2ltV3JJNmxVdEJWMW1WK0pYalBHT2RyUi9nVUpEZFNaei9paWl5L3Z6eUtNMm9IZ2hDSlZvSEZGcXlqTmhKV2d4QmgyYjlhR2VkUGcvbVVlZStHT29wak9zbWg0Q0g2MDhBbXlibkFWUXlZZGdYOEVhQ1ppckY1enlPUjZUM05JU050Q3o5b3lvMnpHRGJad3c9PQ==\"},{\"name\":\"X-SES-DKIM-SIGNATURE\",\"value\":\"a=rsa-sha256; q=dns/txt; b=azfbEksYORI2UHG1unhG3Lv+PKFEbkyjxwQjM940Lb1HPrIthv4Ip3Q4tjqsadGt09/NsEz/pZ5eU9mwFPwtKQOuTyvKe52BMD2rnBcCpdpWetHqPqxv7vMgCS10eInfQSipSibMJRzhGBTQ/l/wV+1zlGywIpVue+md5ySyL0I=; c=relaxed/simple; s=ug7nbtf4gccmlpwj322ax3p6ow6yfsug; d=amazonses.com; t=1735618556; v=1; bh=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN/XKdLCPjaYaY=; h=From:To:Cc:Bcc:Subject:Date:Message-ID:MIME-Version:Content-Type:X-SES-RECEIPT;\"},{\"name\":\"DKIM-Signature\",\"value\":\"v=1; a=rsa-sha256; c=relaxed/relaxed; d=johnjcsmith.com; s=sig1; t=1735618554; bh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=; h=From:Content-Type:Mime-Version:Subject:Message-Id:Date:To:x-icloud-hme; b=okadMTLg+ro8E2nndzhy3joxbL5YO33ubsa7NBcQXXxQnGGjQ95gErjP5iAciJAR5/+4LdUiYrfZHg/cLfSR+ZIwZ6/CQsQ3VYZZh5X/klbTLlfNurDo2SwwAAEDwnocwIQ1BF2PCBIbL0yaqmkXKw0+7RxAsKc5pn0dp2rrHhr6d3lLN2sU+EBnKVcZgt0yv7HSe5goghjEmy9+0VcQNwIySZhko4cV7l7po93m46rUY5Wi2k2Yh/vzL777vbLIxNtPwp4eRL46ucea8J9EA91uSHgyLom1TPQyXPIqA7/DYXyolC4xJxUGSNPw45NfYWs4mZAZpmc1sOSf5xHCFA==\"},{\"name\":\"Received\",\"value\":\"from smtpclient.apple (mr38p00im-dlb-asmtp-mailmevip.me.com [17.57.152.18]) by mr85p00im-ztdg06011901.me.com (Postfix) with ESMTPSA id DC5C41349C75 for ; Tue, 31 Dec 2024 04:15:52 +0000 (UTC)\"},{\"name\":\"From\",\"value\":\"john@johnjcsmith.com\"},{\"name\":\"Content-Type\",\"value\":\"text/plain\"},{\"name\":\"Content-Transfer-Encoding\",\"value\":\"7bit\"},{\"name\":\"Mime-Version\",\"value\":\"1.0 (Mac OS X Mail 16.0 \\\\(3826.300.87.4.3\\\\))\"},{\"name\":\"Subject\",\"value\":\"This is a test\"},{\"name\":\"Message-Id\",\"value\":\"<93FC27CD-9054-4BB5-ADA9-C9CB425D3844@johnjcsmith.com>\"},{\"name\":\"Date\",\"value\":\"Tue, 31 Dec 2024 14:45:38 +1030\"},{\"name\":\"To\",\"value\":\"test@run.inferable.ai\"},{\"name\":\"X-Mailer\",\"value\":\"Apple Mail (2.3826.300.87.4.3)\"},{\"name\":\"X-Proofpoint-GUID\",\"value\":\"1yPlQIuJlVJxiCqK-1kin_Sns19OcPdQ\"},{\"name\":\"X-Proofpoint-ORIG-GUID\",\"value\":\"1yPlQIuJlVJxiCqK-1kin_Sns19OcPdQ\"},{\"name\":\"X-Proofpoint-Virus-Version\",\"value\":\"\"},{\"name\":\"X-Proofpoint-Spam-Details\",\"value\":\"rule=notspam policy=default score=0 phishscore=0 malwarescore=0 spamscore=0 bulkscore=0 adultscore=0 mlxlogscore=343 suspectscore=0 mlxscore=0 clxscore=1030 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.19.0-2308100000 definitions=main-2412310033\"}],\"commonHeaders\":{\"returnPath\":\"john@johnjcsmith.com\",\"from\":[\"john@johnjcsmith.com\"],\"date\":\"Tue, 31 Dec 2024 14:45:38 +1030\",\"to\":[\"test@run.inferable.ai\"],\"messageId\":\"<93FC27CD-9054-4BB5-ADA9-C9CB425D3844@johnjcsmith.com>\",\"subject\":\"This is a test\"}},\"receipt\":{\"timestamp\":\"2024-12-31T04:15:55.348Z\",\"processingTimeMillis\":764,\"recipients\":[\"test@run.inferable.ai\"],\"spamVerdict\":{\"status\":\"PASS\"},\"virusVerdict\":{\"status\":\"PASS\"},\"spfVerdict\":{\"status\":\"PASS\"},\"dkimVerdict\":{\"status\":\"FAIL\"},\"dmarcVerdict\":{\"status\":\"GRAY\"},\"action\":{\"type\":\"SNS\",\"topicArn\":\"arn:aws:sns:us-east-1:590183853800:email-ingestion-topic\",\"encoding\":\"UTF8\"}},\"content\":\"Return-Path: \\r\\nReceived: from mr85p00im-ztdg06011901.me.com (mr85p00im-ztdg06011901.me.com [17.58.23.198])\\r\\n by inbound-smtp.us-east-1.amazonaws.com with SMTP id io5akr05ppl6s5095kjm6p5rusbbsaode3gn6j81\\r\\n for test@run.inferable.ai;\\r\\n Tue, 31 Dec 2024 04:15:55 +0000 (UTC)\\r\\nX-SES-Spam-Verdict: PASS\\r\\nX-SES-Virus-Verdict: PASS\\r\\nReceived-SPF: pass (spfCheck: domain of icloud.com designates 17.58.23.198 as permitted sender) client-ip=17.58.23.198; envelope-from=john@johnjcsmith.com; helo=mr85p00im-ztdg06011901.me.com;\\r\\nAuthentication-Results: amazonses.com;\\r\\n spf=pass (spfCheck: domain of icloud.com designates 17.58.23.198 as permitted sender) client-ip=17.58.23.198; envelope-from=john@johnjcsmith.com; helo=mr85p00im-ztdg06011901.me.com;\\r\\n dkim=fail header.i=@johnjcsmith.com;\\r\\n dmarc=none header.from=johnjcsmith.com;\\r\\nX-SES-RECEIPT: AEFBQUFBQUFBQUFHWTBhd3Fsa3ZBdE9GNnJUbWw0ZDFEemdKSlRuNUxNMWkveGFwMGtiVTBzanVNdWE5eEhodFhHM1MrWVhkYnVSN2FDVFNGQ0NIdlQ4V1pjdDVOQVVQL2dPMU9nUHQ0a1dQbEdRdUJqZE94NWt3Ri9XTEtXWlllYkhZVW9TTkZsUHZITUcxZ0VPb255OU9WT3pWS0dsUkZ5L0I4V2ltV3JJNmxVdEJWMW1WK0pYalBHT2RyUi9nVUpEZFNaei9paWl5L3Z6eUtNMm9IZ2hDSlZvSEZGcXlqTmhKV2d4QmgyYjlhR2VkUGcvbVVlZStHT29wak9zbWg0Q0g2MDhBbXlibkFWUXlZZGdYOEVhQ1ppckY1enlPUjZUM05JU050Q3o5b3lvMnpHRGJad3c9PQ==\\r\\nX-SES-DKIM-SIGNATURE: a=rsa-sha256; q=dns/txt; b=azfbEksYORI2UHG1unhG3Lv+PKFEbkyjxwQjM940Lb1HPrIthv4Ip3Q4tjqsadGt09/NsEz/pZ5eU9mwFPwtKQOuTyvKe52BMD2rnBcCpdpWetHqPqxv7vMgCS10eInfQSipSibMJRzhGBTQ/l/wV+1zlGywIpVue+md5ySyL0I=; c=relaxed/simple; s=ug7nbtf4gccmlpwj322ax3p6ow6yfsug; d=amazonses.com; t=1735618556; v=1; bh=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN/XKdLCPjaYaY=; h=From:To:Cc:Bcc:Subject:Date:Message-ID:MIME-Version:Content-Type:X-SES-RECEIPT;\\r\\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=johnjcsmith.com;\\r\\n\\ts=sig1; t=1735618554;\\r\\n\\tbh=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=;\\r\\n\\th=From:Content-Type:Mime-Version:Subject:Message-Id:Date:To:\\r\\n\\t x-icloud-hme;\\r\\n\\tb=okadMTLg+ro8E2nndzhy3joxbL5YO33ubsa7NBcQXXxQnGGjQ95gErjP5iAciJAR5\\r\\n\\t /+4LdUiYrfZHg/cLfSR+ZIwZ6/CQsQ3VYZZh5X/klbTLlfNurDo2SwwAAEDwnocwIQ\\r\\n\\t 1BF2PCBIbL0yaqmkXKw0+7RxAsKc5pn0dp2rrHhr6d3lLN2sU+EBnKVcZgt0yv7HSe\\r\\n\\t 5goghjEmy9+0VcQNwIySZhko4cV7l7po93m46rUY5Wi2k2Yh/vzL777vbLIxNtPwp4\\r\\n\\t eRL46ucea8J9EA91uSHgyLom1TPQyXPIqA7/DYXyolC4xJxUGSNPw45NfYWs4mZAZp\\r\\n\\t mc1sOSf5xHCFA==\\r\\nReceived: from smtpclient.apple (mr38p00im-dlb-asmtp-mailmevip.me.com [17.57.152.18])\\r\\n\\tby mr85p00im-ztdg06011901.me.com (Postfix) with ESMTPSA id DC5C41349C75\\r\\n\\tfor ; Tue, 31 Dec 2024 04:15:52 +0000 (UTC)\\r\\nFrom: john@johnjcsmith.com\\r\\nContent-Type: text/plain\\r\\nContent-Transfer-Encoding: 7bit\\r\\nMime-Version: 1.0 (Mac OS X Mail 16.0 \\\\(3826.300.87.4.3\\\\))\\r\\nSubject: This is a test\\r\\nMessage-Id: <93FC27CD-9054-4BB5-ADA9-C9CB425D3844@johnjcsmith.com>\\r\\nDate: Tue, 31 Dec 2024 14:45:38 +1030\\r\\nTo: test@run.inferable.ai\\r\\nX-Mailer: Apple Mail (2.3826.300.87.4.3)\\r\\nX-Proofpoint-GUID: 1yPlQIuJlVJxiCqK-1kin_Sns19OcPdQ\\r\\nX-Proofpoint-ORIG-GUID: 1yPlQIuJlVJxiCqK-1kin_Sns19OcPdQ\\r\\nX-Proofpoint-Virus-Version:\\r\\nX-Proofpoint-Spam-Details: rule=notspam policy=default score=0 phishscore=0 malwarescore=0 spamscore=0\\r\\n bulkscore=0 adultscore=0 mlxlogscore=343 suspectscore=0 mlxscore=0\\r\\n clxscore=1030 classifier=spam adjust=0 reason=mlx scancount=1\\r\\n engine=8.19.0-2308100000 definitions=main-2412310033\\r\\n\\r\\n\"}", - "Timestamp" : "2024-12-31T04:15:56.138Z", - "SignatureVersion" : "1", - "Signature" : "SOtyVndg0O6ldUwxRYsYORfRQF2mjz8HoEBGrR79bqO1Py7wLFJgSVABQFBVxEIypZZ6HGRFw57FT7V5BnSM5zbKfep0ERyYhcBaSHg9dW9bPsDYZqDHxXxcjlILOYepALZ9Fiy2Trk5Rw2bFZMsbsgIIXx/YcuFKw3zYwq7r062fVm6+y3EHgS8fK5w88cGjChEM4ktbCUQKWtzdQYdOLLlHmAnf8pScsVx7M2SpCdfZRprfNWgmiNDXvDpSHCYtcoi2iEVMtLYDxGFtE0m9ZJjbIOPTlowEkAp/m/y/6MB0/+Fvv1UbOghjpgUiIESu8CWMpcwKgZvYLcDsMNqCg==", - "SigningCertURL" : "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-9c6465fa7f48f5cacd23014631ec1136.pem", - "UnsubscribeURL" : "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:590183853800:email-ingestion-topic:1b1ec808-c78a-43fc-a08a-e19b3be0cd7d" -} -EOF -) - -aws --region=us-west-2 --endpoint=http://localhost:9324 sqs send-message --queue-url http://localhost:9324/000000000000/email-ingestion --message-body "$MESSAGE_BODY" diff --git a/control-plane/scripts/sendTestEmail.ts b/control-plane/scripts/sendTestEmail.ts new file mode 100755 index 00000000..f52559e2 --- /dev/null +++ b/control-plane/scripts/sendTestEmail.ts @@ -0,0 +1,30 @@ +#!/usr/bin/env tsx + +import { exec } from "child_process"; +import { buildMessageBody } from "../src/modules/email/test.util"; + +const messageBody = JSON.stringify(buildMessageBody({ + body: "What tools are available", + from: "john@johnjcsmith.com", + subject: "Subject", + to: [ "01JE9SYD010WFFJ6GVE26WGVVK@run.inferable.ai" ] +})); + +// Define the AWS CLI command +const region = "us-west-2"; +const endpoint = "http://localhost:9324"; +const queueUrl = "http://localhost:9324/000000000000/email-ingestion"; +const command = `aws --region=${region} --endpoint=${endpoint} sqs send-message --queue-url ${queueUrl} --message-body '${messageBody}'`; + +// Execute the AWS CLI command +exec(command, (error, stdout, stderr) => { + if (error) { + console.error(`Error executing command: ${error.message}`); + return; + } + if (stderr) { + console.error(`stderr: ${stderr}`); + return; + } + console.log(`stdout: ${stdout}`); +}); diff --git a/control-plane/src/modules/clerk.ts b/control-plane/src/modules/clerk.ts index efbf328e..4d355761 100644 --- a/control-plane/src/modules/clerk.ts +++ b/control-plane/src/modules/clerk.ts @@ -9,7 +9,7 @@ const getOrgIdForClusterId = async (cluserId: string) => { organizationId: clusters.organization_id }).from(clusters).where(eq(clusters.id, cluserId)); - return cluster.organizationId + return cluster?.organizationId } const getUserForOrg = async (emailAddress: string, organizationId: string) => { @@ -30,12 +30,12 @@ const getUserForOrg = async (emailAddress: string, organizationId: string) => { export const getUserForCluster = async ({ emailAddress, - cluserId + clusterId }: { emailAddress: string; - cluserId: string; + clusterId: string; }) => { - const organizationId = await getOrgIdForClusterId(cluserId); + const organizationId = await getOrgIdForClusterId(clusterId); if (!organizationId) { throw new Error("Can not lookup user without organizationId") diff --git a/control-plane/src/modules/email/index.test.ts b/control-plane/src/modules/email/index.test.ts new file mode 100644 index 00000000..5cd318d7 --- /dev/null +++ b/control-plane/src/modules/email/index.test.ts @@ -0,0 +1,49 @@ +import { ulid } from "ulid"; +import { parseMessage } from "."; +import { createOwner } from "../test/util"; +import { buildMessageBody } from "./test.util"; + + +describe("parseMessage", () => { + const clusterId = ulid(); + const organizationId = ulid(); + + beforeAll(async () => { + await createOwner({ + clusterId, + organizationId, + }); + }) + + it("should parse a message ingestion event", async () => { + const result = await parseMessage(buildMessageBody({ + from: "test@example.com", + to: [ `${clusterId}@run.inferable.ai` ], + subject: "Subject", + body: "What tools are available" + })); + + expect(result).toBeDefined(); + expect(result.clusterId).toBe(clusterId); + expect(result.source).toBe("test@example.com"); + }); + + it("should fail with multiple '@run.inferable.ai' addresses", async () => { + await expect(parseMessage(buildMessageBody({ + from: "test@example.com", + to: [ "12343@run.inferable.ai", `${clusterId}@run.inferable.ai` ], + subject: "Subject", + body: "What tools are available" + }))).rejects.toThrow("Found multiple Inferable email addresses in destination"); + }) + + it("should fail with no '@run.inferable.ai' addresses", async () => { + await expect(parseMessage(buildMessageBody({ + from: "test@example.com", + to: [ "something@else.com" ], + subject: "Subject", + body: "What tools are available" + }))).rejects.toThrow("Could not extract clusterId from email address"); + }) + +}) diff --git a/control-plane/src/modules/email/index.ts b/control-plane/src/modules/email/index.ts index 12735f19..efb396c4 100644 --- a/control-plane/src/modules/email/index.ts +++ b/control-plane/src/modules/email/index.ts @@ -1,15 +1,19 @@ import { Consumer } from "sqs-consumer"; import { env } from "../../utilities/env"; -import { sqs } from "../sqs"; +import { BaseMessage, sqs, withObservability } from "../sqs"; import { z } from "zod"; -import { Message } from "@aws-sdk/client-sqs"; import { logger } from "../observability/logger"; import { safeParse } from "../../utilities/safe-parse"; +import { ParsedMail, simpleParser } from "mailparser"; +import { getUserForCluster } from "../clerk"; +import { AuthenticationError } from "../../utilities/errors"; +import { createRunWithMessage } from "../workflows/workflows"; +import { flagsmith } from "../flagsmith"; const sesMessageSchema = z.object({ notificationType: z.string(), mail: z.object({ - timestamp: z.string().datetime(), + timestamp: z.string(), source: z.string().email(), messageId: z.string(), destination: z.array(z.string().email()), @@ -30,7 +34,7 @@ const sesMessageSchema = z.object({ }), }), receipt: z.object({ - timestamp: z.string().datetime(), + timestamp: z.string(), processingTimeMillis: z.number(), recipients: z.array(z.string().email()), spamVerdict: z.object({ status: z.string() }), @@ -54,11 +58,11 @@ const snsNotificationSchema = z.object({ TopicArn: z.string(), Subject: z.string(), Message: z.string(), - Timestamp: z.string().datetime(), + Timestamp: z.string(), SignatureVersion: z.string(), Signature: z.string(), - SigningCertURL: z.string().url(), - UnsubscribeURL: z.string().url(), + SigningCertURL: z.string(), + UnsubscribeURL: z.string(), }); const emailIngestionConsumer = env.SQS_EMAIL_INGESTION_QUEUE_URL @@ -67,7 +71,7 @@ const emailIngestionConsumer = env.SQS_EMAIL_INGESTION_QUEUE_URL batchSize: 5, visibilityTimeout: 60, heartbeatInterval: 30, - handleMessage: handleEmailIngestion, + handleMessage: withObservability(env.SQS_EMAIL_INGESTION_QUEUE_URL, handleEmailIngestion), sqs, }) : undefined; @@ -80,52 +84,132 @@ export const stop = async () => { emailIngestionConsumer?.stop(); }; -async function handleEmailIngestion(message: Message) { - try { - const notificationJson = safeParse(message.Body); - if (!notificationJson.success) { - logger.error("SNS notification is not valid JSON", { - error: notificationJson.error, - }); - return; - } - - const notification = snsNotificationSchema.safeParse(notificationJson.data); - if (!notification.success) { - logger.error("Could not parse SNS notification", { - error: notification.error, - }); - return; - } - - - const sesJson = safeParse(notification.data.Message); - if (!sesJson.success) { - logger.error("SES message is not valid JSON", { - error: sesJson.error, - }); - return; - } - - const sesMessage = sesMessageSchema.safeParse(sesJson.data); - if (!sesMessage.success) { - logger.error("Could not parse SES message", { - error: sesMessage.error, - }); - return; - } - - logger.info("Ingesting email event", { - messageId: sesMessage.data.mail.messageId, - source: sesMessage.data.mail.source, - destination: sesMessage.data.mail.destination, - subject: sesMessage.data.mail.commonHeaders.subject, +async function handleEmailIngestion(raw: unknown) { + const message = await parseMessage(raw); + if (!message.body) { + logger.info("Email had no body. Skipping", { }); + return; + } + + const user = await authenticateUser(message.source, message.clusterId); + + const flags = await flagsmith?.getIdentityFlags(message.clusterId, { + clusterId: message.clusterId, + }); + + const useEmail = flags?.isFeatureEnabled("experimental_email_trigger"); + + if (!useEmail) { + logger.info("Email trigger is disabled. Skipping", { + clusterId: message.clusterId, + }); + return; + } + + await handleNewChain({ + userId: user.id, + body: message.body, + clusterId: message.clusterId + }); +} + +export async function parseMessage(message: unknown) { + const notification = snsNotificationSchema.safeParse(message); + if (!notification.success) { + throw new Error("Could not parse SNS notification message"); + } + + const sesJson = safeParse(notification.data.Message); + if (!sesJson.success) { + throw new Error("SES message is not valid JSON"); + } + + const sesMessage = sesMessageSchema.safeParse(sesJson.data); + if (!sesMessage.success) { + throw new Error("Could not parse SES message"); + } + + const ingestionAddresses = sesMessage.data.mail.destination.filter( + (email) => email.endsWith(env.INFERABLE_EMAIL_DOMAIN) + ) + + if (ingestionAddresses.length > 1) { + throw new Error("Found multiple Inferable email addresses in destination"); + } + + const clusterId = ingestionAddresses.pop()?.replace(env.INFERABLE_EMAIL_DOMAIN, "").replace("@", ""); + if (!clusterId) { + throw new Error("Could not extract clusterId from email address"); + } + + const mail = await parseMailContent(sesMessage.data.content); + if (!mail) { + throw new Error("Could not parse email content"); + } + + let body = mail.text + if (!body && mail.html) { + body = mail.html + } - } catch (error) { - logger.error("Error while ingesting email event", { - error, + return { + body, + clusterId, + ingestionAddresses, + source: sesMessage.data.mail.source, + messageId: sesMessage.data.mail.messageId, + } +} + +const parseMailContent = (message: string): Promise => { + return new Promise((resolve, reject) => { + simpleParser(message, (error, parsed) => { + if (error) { + reject(error); + } else { + resolve(parsed); + } }); + }) +}; + + +const authenticateUser = async (emailAddress: string, clusterId: string) => { + if (!env.CLERK_SECRET_KEY) { + throw new Error("CLERK_SECRET_KEY must be set for email authentication"); + } + + const clerkUser = await getUserForCluster({ + emailAddress, + clusterId, + }); + + if (!clerkUser) { + logger.info("Could not find Email in Clerk.", { + emailAddress, + }); + throw new AuthenticationError("Could not authenticate Email sender"); } + + return clerkUser; +}; + +const handleNewChain = async ({ + userId, + body, + clusterId, +}: { + userId: string; + body: string; + clusterId: string; +}) => { + await createRunWithMessage({ + userId, + clusterId, + message: body, + type: "human", + }) } + diff --git a/control-plane/src/modules/email/test.util.ts b/control-plane/src/modules/email/test.util.ts new file mode 100644 index 00000000..b3e57263 --- /dev/null +++ b/control-plane/src/modules/email/test.util.ts @@ -0,0 +1,68 @@ +import { ulid } from "ulid"; + +export function buildMessageBody({from, to, subject, body}: { + from: string; + to: string[]; + subject: string; + body: string;}) { + + const messageBase = { + notificationType: "Received", + mail: { + timestamp: "", + source: from, + messageId: "", + destination: to, + headersTruncated: false, + headers: [], + commonHeaders: { + returnPath: from, + from: [ from ], + date: "", + to, + messageId: "<93FC27CD-9054-4BB5-ADA9-C9CB425D3844@johnjcsmith.com>", + subject, + } + }, + receipt: { + timestamp: "", + processingTimeMillis: 764, + recipients: to, + spamVerdict: { status: 'PASS' }, + virusVerdict: { status: 'PASS' }, + spfVerdict: { status: 'PASS' }, + dkimVerdict: { status: 'FAIL' }, + dmarcVerdict: { status: 'GRAY' }, + action: { + type: "", + topicArn: "", + encoding: 'UTF8' + } + }, + content: `Return-Path: <${from}>\r\n` + + `From: ${from}\r\n` + + 'Content-Type: text/plain\r\n' + + 'Content-Transfer-Encoding: 7bit\r\n' + + 'Mime-Version: 1.0 (Mac OS X Mail 16.0 \\(3826.300.87.4.3\\))\r\n' + + 'Subject: ${subject}\r\n' + + 'Message-Id: <93FC27CD-9054-4BB5-ADA9-C9CB425D3844@johnjcsmith.com>\r\n' + + 'Date: Tue, 31 Dec 2024 14:45:38 +1030\r\n' + + `To: ${to}\r\n` + + '\r\n' + + body + + '\r\n' + } + + return { + Type: "Notification", + MessageId: ulid(), + TopicArn: "", + Subject: "", + Timestamp: "", + SignatureVersion: "", + Signature: "", + SigningCertURL: "", + UnsubscribeURL: "", + Message: JSON.stringify(messageBase) + } +} diff --git a/control-plane/src/modules/integrations/slack/index.ts b/control-plane/src/modules/integrations/slack/index.ts index 01e181bf..6ad86bb3 100644 --- a/control-plane/src/modules/integrations/slack/index.ts +++ b/control-plane/src/modules/integrations/slack/index.ts @@ -556,7 +556,7 @@ const authenticateUser = async (userId: string, client: webApi.WebClient, integr const clerkUser = await getUserForCluster({ emailAddress: email, - cluserId: integration.cluster_id, + clusterId: integration.cluster_id, }); if (!clerkUser) { diff --git a/control-plane/src/modules/jobs/external.ts b/control-plane/src/modules/jobs/external.ts index fe878252..31cf4b5c 100644 --- a/control-plane/src/modules/jobs/external.ts +++ b/control-plane/src/modules/jobs/external.ts @@ -1,6 +1,6 @@ import { Consumer } from "sqs-consumer"; import { env } from "../../utilities/env"; -import { BaseMessage, baseMessageSchema, sqs, withObservability } from "../sqs"; +import { baseMessageSchema, sqs, withObservability } from "../sqs"; import { z } from "zod"; import { logger } from "../observability/logger"; import { getJob } from "./jobs"; @@ -26,7 +26,7 @@ export const stop = async () => { externalCallConsumer?.stop(); }; -async function handleExternalCall(message: BaseMessage) { +async function handleExternalCall(message: unknown) { const zodResult = baseMessageSchema .extend({ callId: z.string(), diff --git a/control-plane/src/modules/sqs.ts b/control-plane/src/modules/sqs.ts index 0dc1636b..e7a35a59 100644 --- a/control-plane/src/modules/sqs.ts +++ b/control-plane/src/modules/sqs.ts @@ -24,7 +24,7 @@ export const baseMessageSchema = z export type BaseMessage = z.infer; export const withObservability = - (queueUrl: string, fn: (message: BaseMessage) => Promise) => + (queueUrl: string, fn: (message: unknown) => Promise) => async (message: Message) => { const jsonResult = safeParse(message.Body); if (!jsonResult.success) { @@ -35,7 +35,12 @@ export const withObservability = return; } - const zodResult = baseMessageSchema.safeParse(jsonResult.data); + const zodResult = baseMessageSchema + .extend({ + clusterId: z.string().optional(), + runId: z.string().optional(), + }) + .safeParse(jsonResult.data); if (!zodResult.success) { logger.error("Message body does conform to base schema", { diff --git a/control-plane/src/modules/workflows/queues.ts b/control-plane/src/modules/workflows/queues.ts index 20a87892..0cbdc7b0 100644 --- a/control-plane/src/modules/workflows/queues.ts +++ b/control-plane/src/modules/workflows/queues.ts @@ -39,7 +39,7 @@ export const stop = async () => { }; const MAX_PROCESS_LOCK_ATTEMPTS = 5; -async function handleRunProcess(message: BaseMessage) { +async function handleRunProcess(message: unknown) { const zodResult = baseMessageSchema .extend({ lockAttempts: z.number().default(0), @@ -104,7 +104,7 @@ async function handleRunProcess(message: BaseMessage) { } } -async function handleRunNameGeneration(message: BaseMessage) { +async function handleRunNameGeneration(message: unknown) { const zodResult = baseMessageSchema .extend({ content: z.string(), diff --git a/control-plane/src/modules/workflows/workflows.ts b/control-plane/src/modules/workflows/workflows.ts index bfdd4b62..fc420088 100644 --- a/control-plane/src/modules/workflows/workflows.ts +++ b/control-plane/src/modules/workflows/workflows.ts @@ -12,7 +12,6 @@ import { import { ulid } from "ulid"; import { env } from "../../utilities/env"; import { BadRequestError, NotFoundError, RunBusyError } from "../../utilities/errors"; -import { Auth } from "../auth/auth"; import { clusters, db, diff --git a/control-plane/src/utilities/env.ts b/control-plane/src/utilities/env.ts index cfc9840d..aeb6ab71 100644 --- a/control-plane/src/utilities/env.ts +++ b/control-plane/src/utilities/env.ts @@ -67,6 +67,8 @@ const envSchema = z BEDROCK_AVAILABLE: truthy.default(false), + INFERABLE_EMAIL_DOMAIN: z.string().default("run.inferable.ai"), + // Observability HYPERDX_API_KEY: z.string().optional(), ROLLBAR_ACCESS_TOKEN: z.string().optional(),