From 96a5aed5048c342a39e54ebbc14216df8053058d Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Sun, 5 Nov 2023 22:18:13 -0500 Subject: [PATCH] use imap for attachments --- package-lock.json | 289 ++++++++++++++++++++++++++++++++++++++++ package.json | 2 + tools/download-stats.js | 140 +++++++++++++++++-- 3 files changed, 419 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 41e3c3ad..a6c2118a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,8 +24,10 @@ "express": "^4.18.2", "fs-extra": "^11.1.1", "glob": "^10.3.10", + "imap": "^0.8.19", "ip": "^1.1.8", "json5": "^2.2.3", + "mailparser": "^3.6.5", "roku-deploy": "^3.10.3", "rooibos-roku": "^5.8.0", "semver": "^7.5.4", @@ -305,6 +307,19 @@ "node": ">=12" } }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "dev": true, + "dependencies": { + "domhandler": "^5.0.3", + "selderee": "^0.11.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -1603,6 +1618,15 @@ "node": ">=0.10.0" } }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1761,6 +1785,15 @@ "iconv-lite": "^0.6.2" } }, + "node_modules/encoding-japanese": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.0.0.tgz", + "integrity": "sha512-++P0RhebUC8MJAwJOsT93dT+5oc5oPImp1HubZpAuCZ5kTLnhuuBhKHj2jJeO/Gj93idPBWmIuQ9QWMe5rX3pQ==", + "dev": true, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/encoding/node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -2382,6 +2415,31 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, + "node_modules/html-to-text": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "dev": true, + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.11.0", + "deepmerge": "^4.3.1", + "dom-serializer": "^2.0.0", + "htmlparser2": "^8.0.2", + "selderee": "^0.11.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/htmlparser2": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", @@ -2499,6 +2557,43 @@ } ] }, + "node_modules/imap": { + "version": "0.8.19", + "resolved": "https://registry.npmjs.org/imap/-/imap-0.8.19.tgz", + "integrity": "sha512-z5DxEA1uRnZG73UcPA4ES5NSCGnPuuouUx43OPX7KZx1yzq3N8/vx2mtXEShT5inxB3pRgnfG1hijfu7XN2YMw==", + "dev": true, + "dependencies": { + "readable-stream": "1.1.x", + "utf7": ">=1.0.2" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/imap/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "dev": true + }, + "node_modules/imap/node_modules/readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/imap/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", + "dev": true + }, "node_modules/immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", @@ -2752,6 +2847,51 @@ "lcov-parse": "bin/cli.js" } }, + "node_modules/leac": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", + "dev": true, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/libbase64": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.2.1.tgz", + "integrity": "sha512-l+nePcPbIG1fNlqMzrh68MLkX/gTxk/+vdvAb388Ssi7UuUN31MI44w4Yf33mM3Cm4xDfw48mdf3rkdHszLNew==", + "dev": true + }, + "node_modules/libmime": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.2.1.tgz", + "integrity": "sha512-A0z9O4+5q+ZTj7QwNe/Juy1KARNb4WaviO4mYeFC4b8dBT2EEqK2pkM+GC8MVnkOjqhl5nYQxRgnPYRRTNmuSQ==", + "dev": true, + "dependencies": { + "encoding-japanese": "2.0.0", + "iconv-lite": "0.6.3", + "libbase64": "1.2.1", + "libqp": "2.0.1" + } + }, + "node_modules/libmime/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libqp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.0.1.tgz", + "integrity": "sha512-Ka0eC5LkF3IPNQHJmYBWljJsw0UvM6j+QdKRbWyCdTmYwvIDE6a7bCm0UkTAL/K+3KXK5qXT/ClcInU01OpdLg==", + "dev": true + }, "node_modules/lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -2761,6 +2901,15 @@ "immediate": "~3.0.5" } }, + "node_modules/linkify-it": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-4.0.1.tgz", + "integrity": "sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==", + "dev": true, + "dependencies": { + "uc.micro": "^1.0.1" + } + }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -2824,6 +2973,70 @@ "node": ">=12" } }, + "node_modules/mailparser": { + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/mailparser/-/mailparser-3.6.5.tgz", + "integrity": "sha512-nteTpF0Khm5JLOnt4sigmzNdUH/6mO7PZ4KEnvxf4mckyXYFFhrtAWZzbq/V5aQMH+049gA7ZjfLdh+QiX2Uqg==", + "dev": true, + "dependencies": { + "encoding-japanese": "2.0.0", + "he": "1.2.0", + "html-to-text": "9.0.5", + "iconv-lite": "0.6.3", + "libmime": "5.2.1", + "linkify-it": "4.0.1", + "mailsplit": "5.4.0", + "nodemailer": "6.9.3", + "tlds": "1.240.0" + } + }, + "node_modules/mailparser/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mailsplit": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/mailsplit/-/mailsplit-5.4.0.tgz", + "integrity": "sha512-wnYxX5D5qymGIPYLwnp6h8n1+6P6vz/MJn5AzGjZ8pwICWssL+CCQjWBIToOVHASmATot4ktvlLo6CyLfOXWYA==", + "dev": true, + "dependencies": { + "libbase64": "1.2.1", + "libmime": "5.2.0", + "libqp": "2.0.1" + } + }, + "node_modules/mailsplit/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mailsplit/node_modules/libmime": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.2.0.tgz", + "integrity": "sha512-X2U5Wx0YmK0rXFbk67ASMeqYIkZ6E5vY7pNWRKtnNzqjvdYYG8xtPDpCnuUEnPU9vlgNev+JoSrcaKSUaNvfsw==", + "dev": true, + "dependencies": { + "encoding-japanese": "2.0.0", + "iconv-lite": "0.6.3", + "libbase64": "1.2.1", + "libqp": "2.0.1" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -2980,6 +3193,15 @@ } } }, + "node_modules/nodemailer": { + "version": "6.9.3", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.3.tgz", + "integrity": "sha512-fy9v3NgTzBngrMFkDsKEj0r02U7jm6XfC3b52eoNV+GCrGj+s8pt5OqhiJdWKuw51zCTdiNR/IUD1z33LIIGpg==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -3159,6 +3381,19 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parseley": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "dev": true, + "dependencies": { + "leac": "^0.6.0", + "peberminta": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -3241,6 +3476,15 @@ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", "dev": true }, + "node_modules/peberminta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", + "dev": true, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -3846,6 +4090,18 @@ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "dev": true }, + "node_modules/selderee": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "dev": true, + "dependencies": { + "parseley": "^0.12.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -4213,6 +4469,15 @@ "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "dev": true }, + "node_modules/tlds": { + "version": "1.240.0", + "resolved": "https://registry.npmjs.org/tlds/-/tlds-1.240.0.tgz", + "integrity": "sha512-1OYJQenswGZSOdRw7Bql5Qu7uf75b+F3HFBXbqnG/ifHa0fev1XcG+3pJf3pA/KC6RtHQzfKgIf1vkMlMG7mtQ==", + "dev": true, + "bin": { + "tlds": "bin.js" + } + }, "node_modules/tmp": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", @@ -4339,6 +4604,12 @@ "node": ">=14.17" } }, + "node_modules/uc.micro": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", + "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", + "dev": true + }, "node_modules/unbzip2-stream": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", @@ -4398,6 +4669,24 @@ "requires-port": "^1.0.0" } }, + "node_modules/utf7": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/utf7/-/utf7-1.0.2.tgz", + "integrity": "sha512-qQrPtYLLLl12NF4DrM9CvfkxkYI97xOb5dsnGZHE3teFr0tWiEZ9UdgMPczv24vl708cYMpe6mGXGHrotIp3Bw==", + "dev": true, + "dependencies": { + "semver": "~5.3.0" + } + }, + "node_modules/utf7/node_modules/semver": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "integrity": "sha512-mfmm3/H9+67MCVix1h+IXTpDwL6710LyHuk7+cWC9T1mE0qz4iHhh6r4hU2wrIT9iTsAAC2XQRvfblL028cpLw==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index b552b413..38a7f90a 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,10 @@ "express": "^4.18.2", "fs-extra": "^11.1.1", "glob": "^10.3.10", + "imap": "^0.8.19", "ip": "^1.1.8", "json5": "^2.2.3", + "mailparser": "^3.6.5", "roku-deploy": "^3.10.3", "rooibos-roku": "^5.8.0", "semver": "^7.5.4", diff --git a/tools/download-stats.js b/tools/download-stats.js index 41b63b5e..1a7e72d2 100644 --- a/tools/download-stats.js +++ b/tools/download-stats.js @@ -1,22 +1,138 @@ -const fs = require('fs') +const fs = require('fs'); +const path = require('path'); +const Imap = require('imap'); +const { promisify } = require('util'); +const { simpleParser } = require('mailparser'); +const getEnvVars = require('./get-env-vars'); -const markdownFile = "./playlet.wiki/Home.md" +const attachmentDestination = './playlet.wiki'; + +function writeMarkDownFile(images) { + const markdownFile = path.join(attachmentDestination, 'Home.md'); + + let markdownContent = `# Playlet stats -const markdownContent = ` -# Playlet stats This page was automatically generated on ${new Date().toUTCString()}. -## Channel Health +`; + images.forEach(image => { + markdownContent += `## ${image.title} + +![${image.title}](${image.filename}) + +`; + }); + + fs.writeFileSync(markdownFile, markdownContent); +} + +const config = getEnvVars(['EMAIL', 'EMAIL_APP_PASSWORD']); + +const imap = new Imap({ + user: config.EMAIL, + password: config.EMAIL_APP_PASSWORD, + host: 'imap.gmail.com', + port: 993, + tls: true, + tlsOptions: { + rejectUnauthorized: false + } +}); + +const searchAsync = promisify(imap.search).bind(imap); +const openBoxAsync = promisify(imap.openBox).bind(imap); + +async function getAttachementAsync(from, subject, since) { + return new Promise(async (resolve, reject) => { + const results = await searchAsync([['FROM', from], ['SUBJECT', subject], ['SINCE', since]]); + const fetch = imap.fetch(results, { bodies: '' }); + + let messageCount = 0; + + fetch.on('message', (msg, seqno) => { + console.log('Message #%d', seqno); + messageCount++; + + let data = "" + msg.on("body", function (stream) { + stream.on("data", function (chunk) { + data = data + chunk.toString("utf8"); + }); + + stream.once("end", async () => { + console.log('stream end - parsing...'); + try { + const parser = await simpleParser(data); + const attachments = parser.attachments; + if (attachments.length !== 1) { + throw new Error(`Expected 1 attachment, but found ${attachments.length}`); + } + const attachment = attachments[0]; + const filename = attachment.filename; + fs.writeFileSync(path.join(attachmentDestination, filename), attachment.content); + console.log('attachment written'); + resolve(filename); + } catch (error) { + console.error('Error:', error); + reject(error); + } + }) + }); + msg.once("end", function () { + console.log("Finished msg #" + seqno); + }); + }); + + fetch.once('end', () => { + console.log('fetch end'); + if (messageCount === 0) { + reject(new Error('No message found')); + } + }); + }); +} +const timeout = setTimeout(() => { + console.log('Timeout'); + process.exit(1); +}, 10 * 60 * 1000); + +imap.once('ready', async () => { + try { + await openBoxAsync('INBOX'); + + const now = new Date(); + const yesterday = new Date(now - 24 * 60 * 60 * 1000); + const yesterdayString = yesterday.toISOString().slice(0, 19).replace('T', ' '); -![Channel Health](Channel_Health.png) + const images = [] + images.push({ + title: 'Channel Health', + filename: await getAttachementAsync('bdp_noreply@data.roku.com', 'Channel Health', yesterdayString) + }) + images.push({ + title: 'Channel Engagement', + filename: await getAttachementAsync('bdp_noreply@data.roku.com', 'Channel Engagement', yesterdayString) + }) + images.push({ + title: 'Viewership Summary', + filename: await getAttachementAsync('bdp_noreply@data.roku.com', 'Viewership Summary', yesterdayString) + }) -## Channel Engagement + writeMarkDownFile(images); -![Channel Engagement](Channel_Engagement.png) + imap.end(); + } catch (error) { + console.error('Error:', error); + } + clearTimeout(timeout); +}); -## Viewership Summary +imap.once('error', err => { + console.error(err); +}); -![Viewership Summary](Viewership_Summary.png) -` +imap.once('end', () => { + console.log('Connection ended'); +}); -fs.writeFileSync(markdownFile, markdownContent) \ No newline at end of file +imap.connect();