diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c9fbbb6..b820878 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,6 +39,7 @@ jobs: echo "Writing config wile" echo "${{vars.PROD_CONFIG}}" > config/production.json + echo "${{secrets.ROLLBAR_SERVER_TOKEN}}" > .env echo "Renaming current app folder" mv $appDir $appDir.$(date +\%Y\%m\%d\%H\%M\%S) diff --git a/.gitignore b/.gitignore index d0f2e6c..4d8670f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ node_modules config/debug.json mainpage.png + +.env diff --git a/package-lock.json b/package-lock.json index 7ac0284..568baac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,10 @@ "license": "MIT", "dependencies": { "config": "^3.3.11", + "dotenv": "^16.4.5", "nedb": "^1.8.0", "puppeteer": "^22.6.1", + "rollbar": "^2.26.4", "yargs": "^17.7.2" }, "devDependencies": { @@ -577,6 +579,11 @@ "node": ">= 10.0.0" } }, + "node_modules/console-polyfill": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/console-polyfill/-/console-polyfill-0.3.0.tgz", + "integrity": "sha512-w+JSDZS7XML43Xnwo2x5O5vxB0ID7T5BdqDtyqT6uiCAX2kZAgcWxNaGqT97tZfSHzfOcvrfsDAodKcJ3UvnXQ==" + }, "node_modules/cosmiconfig": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", @@ -640,6 +647,15 @@ } } }, + "node_modules/decache": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/decache/-/decache-3.1.0.tgz", + "integrity": "sha512-p7D6wJ5EJFFq1CcF2lu1XeqKFLBob8jRQGNAvFLTsV3CbSKBl3VtliAVlUIGz2i9H6kEFnI2Amaft5ZopIG2Fw==", + "optional": true, + "dependencies": { + "find": "^0.2.4" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -676,6 +692,17 @@ "node": ">=6.0.0" } }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -705,6 +732,14 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/error-stack-parser": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", + "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==", + "dependencies": { + "stackframe": "^1.3.4" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -988,6 +1023,15 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/find": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/find/-/find-0.2.9.tgz", + "integrity": "sha512-7a4/LCiInB9xYMnAUEjLilL9FKclwbwK7VlXw+h5jMvT2TDFeYFCHM24O1XdnC/on/hx8mxVO3FTQkyHZnOghQ==", + "optional": true, + "dependencies": { + "traverse-chain": "~0.1.0" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -1368,6 +1412,11 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -1798,6 +1847,11 @@ "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" }, + "node_modules/request-ip": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/request-ip/-/request-ip-3.3.0.tgz", + "integrity": "sha512-cA6Xh6e0fDBBBwH77SLJaJPBmD3nWVAcF9/XAcsrIHdjhFzFiB5aNQFytdjCGPezU3ROwrR11IddKAM08vohxA==" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -1839,6 +1893,41 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rollbar": { + "version": "2.26.4", + "resolved": "https://registry.npmjs.org/rollbar/-/rollbar-2.26.4.tgz", + "integrity": "sha512-JKmrj6riYm9ZPJisgxljgH4uCsvjMHDHXrinDF7aAFaP+eoF51HomVPtLcDTYLsrJ568aKVNLUhedFajONBwSg==", + "dependencies": { + "async": "~3.2.3", + "console-polyfill": "0.3.0", + "error-stack-parser": "^2.0.4", + "json-stringify-safe": "~5.0.0", + "lru-cache": "~2.2.1", + "request-ip": "~3.3.0", + "source-map": "^0.5.7" + }, + "optionalDependencies": { + "decache": "^3.0.5" + } + }, + "node_modules/rollbar/node_modules/async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, + "node_modules/rollbar/node_modules/lru-cache": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.2.4.tgz", + "integrity": "sha512-Q5pAgXs+WEAfoEdw2qKQhNFFhMoFMTYqRVKKUMnzuiR7oKFHS7fWo848cPcTKw+4j/IdN17NyzdhVKgabFV0EA==" + }, + "node_modules/rollbar/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -1941,6 +2030,11 @@ "node": ">=0.10.0" } }, + "node_modules/stackframe": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", + "integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==" + }, "node_modules/streamx": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.16.1.tgz", @@ -2035,6 +2129,12 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, + "node_modules/traverse-chain": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", + "integrity": "sha512-up6Yvai4PYKhpNp5PkYtx50m3KbwQrqDwbuZP/ItyL64YEWHAvH6Md83LFLV/GRSk/BoUVwwgUzX6SOQSbsfAg==", + "optional": true + }, "node_modules/tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", @@ -2649,6 +2749,11 @@ "json5": "^2.2.3" } }, + "console-polyfill": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/console-polyfill/-/console-polyfill-0.3.0.tgz", + "integrity": "sha512-w+JSDZS7XML43Xnwo2x5O5vxB0ID7T5BdqDtyqT6uiCAX2kZAgcWxNaGqT97tZfSHzfOcvrfsDAodKcJ3UvnXQ==" + }, "cosmiconfig": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", @@ -2684,6 +2789,15 @@ "ms": "2.1.2" } }, + "decache": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/decache/-/decache-3.1.0.tgz", + "integrity": "sha512-p7D6wJ5EJFFq1CcF2lu1XeqKFLBob8jRQGNAvFLTsV3CbSKBl3VtliAVlUIGz2i9H6kEFnI2Amaft5ZopIG2Fw==", + "optional": true, + "requires": { + "find": "^0.2.4" + } + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2714,6 +2828,11 @@ "esutils": "^2.0.2" } }, + "dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -2740,6 +2859,14 @@ "is-arrayish": "^0.2.1" } }, + "error-stack-parser": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", + "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==", + "requires": { + "stackframe": "^1.3.4" + } + }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -2942,6 +3069,15 @@ "flat-cache": "^3.0.4" } }, + "find": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/find/-/find-0.2.9.tgz", + "integrity": "sha512-7a4/LCiInB9xYMnAUEjLilL9FKclwbwK7VlXw+h5jMvT2TDFeYFCHM24O1XdnC/on/hx8mxVO3FTQkyHZnOghQ==", + "optional": true, + "requires": { + "traverse-chain": "~0.1.0" + } + }, "find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -3223,6 +3359,11 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" + }, "json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -3548,6 +3689,11 @@ "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" }, + "request-ip": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/request-ip/-/request-ip-3.3.0.tgz", + "integrity": "sha512-cA6Xh6e0fDBBBwH77SLJaJPBmD3nWVAcF9/XAcsrIHdjhFzFiB5aNQFytdjCGPezU3ROwrR11IddKAM08vohxA==" + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3573,6 +3719,38 @@ "glob": "^7.1.3" } }, + "rollbar": { + "version": "2.26.4", + "resolved": "https://registry.npmjs.org/rollbar/-/rollbar-2.26.4.tgz", + "integrity": "sha512-JKmrj6riYm9ZPJisgxljgH4uCsvjMHDHXrinDF7aAFaP+eoF51HomVPtLcDTYLsrJ568aKVNLUhedFajONBwSg==", + "requires": { + "async": "~3.2.3", + "console-polyfill": "0.3.0", + "decache": "^3.0.5", + "error-stack-parser": "^2.0.4", + "json-stringify-safe": "~5.0.0", + "lru-cache": "~2.2.1", + "request-ip": "~3.3.0", + "source-map": "^0.5.7" + }, + "dependencies": { + "async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, + "lru-cache": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.2.4.tgz", + "integrity": "sha512-Q5pAgXs+WEAfoEdw2qKQhNFFhMoFMTYqRVKKUMnzuiR7oKFHS7fWo848cPcTKw+4j/IdN17NyzdhVKgabFV0EA==" + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" + } + } + }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -3635,6 +3813,11 @@ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "optional": true }, + "stackframe": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", + "integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==" + }, "streamx": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.16.1.tgz", @@ -3710,6 +3893,12 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, + "traverse-chain": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", + "integrity": "sha512-up6Yvai4PYKhpNp5PkYtx50m3KbwQrqDwbuZP/ItyL64YEWHAvH6Md83LFLV/GRSk/BoUVwwgUzX6SOQSbsfAg==", + "optional": true + }, "tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", diff --git a/package.json b/package.json index b961f5b..4ba0087 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,10 @@ "license": "MIT", "dependencies": { "config": "^3.3.11", + "dotenv": "^16.4.5", "nedb": "^1.8.0", "puppeteer": "^22.6.1", + "rollbar": "^2.26.4", "yargs": "^17.7.2" }, "type": "module", diff --git a/src/index.js b/src/index.js index 6a30b73..551122d 100644 --- a/src/index.js +++ b/src/index.js @@ -8,6 +8,19 @@ import DealsStorage from './storage/dealsStorage.js'; import SectionsStorage from './storage/sectionsStorage.js'; import ExperiencesStorage from './storage/experiencesStorage.js'; +import Rollbar from 'rollbar'; +import dotenv from 'dotenv'; + +dotenv.config(); + +const ROLLBAR_SERVER_TOKEN = process.env.ROLLBAR_SERVER_TOKEN; + +const rollbar = new Rollbar({ + accessToken: ROLLBAR_SERVER_TOKEN, + captureUncaught: true, + captureUnhandledRejections: true, +}); + const SCRAPE_SECTION_COMMAND = 'scrape-section'; const argv = yargs(hideBin(process.argv)) @@ -30,7 +43,7 @@ async function main() { // Start the browser and create a browser instance const startBrowserHeadless = !isInDebugMode(); const browserInstance = browser.startBrowser(startBrowserHeadless); - const scraperController = new ScraperController(); + const scraperController = new ScraperController(rollbar); let result = []; diff --git a/src/pageController.js b/src/pageController.js index 0c68fa7..961f7e8 100644 --- a/src/pageController.js +++ b/src/pageController.js @@ -1,18 +1,20 @@ import scraperObject from './pageScraper.js'; class ScraperController { - constructor() { - + constructor(rollbar) { + this.rollbar = rollbar; } async _scrape(browserInstance, sectionUrl = '') { this.browser = null; + this.rollbar.log('Starting scraping...'); try { this.browser = await browserInstance; const result = await scraperObject.scraper(this.browser, sectionUrl); return result; } catch (err) { console.log('Could not scrape data => ', err); + this.rollbar.error(err); return {sections: [], items: []}; } }