From 34f0895f9fff31a7151e0a7ac996d1118e62fae6 Mon Sep 17 00:00:00 2001 From: Cyrille Derche Date: Sun, 21 May 2023 12:30:35 +0200 Subject: [PATCH] Implement BullMQ --- package-lock.json | 330 ++++++++++++++++++- package.json | 1 + src/app.ts | 6 +- src/controllers/jobs/createJob.controller.ts | 27 ++ src/controllers/jobs/getJob.controller.ts | 38 +++ src/controllers/jobs/index.ts | 7 + src/index.ts | 1 + src/queues/connection.ts | 9 + src/queues/email.queue.ts | 6 + src/queues/imageProcessing.queue.ts | 6 + src/queues/index.ts | 9 + src/queues/queueByName.ts | 16 + src/routes/index.ts | 10 + src/routes/jobs.route.ts | 10 + src/workers/emailWorker.ts | 42 +++ src/workers/imageProcessingWorker.ts | 41 +++ src/workers/index.ts | 7 + 17 files changed, 552 insertions(+), 14 deletions(-) create mode 100644 src/controllers/jobs/createJob.controller.ts create mode 100644 src/controllers/jobs/getJob.controller.ts create mode 100644 src/controllers/jobs/index.ts create mode 100644 src/queues/connection.ts create mode 100644 src/queues/email.queue.ts create mode 100644 src/queues/imageProcessing.queue.ts create mode 100644 src/queues/index.ts create mode 100644 src/queues/queueByName.ts create mode 100644 src/routes/index.ts create mode 100644 src/routes/jobs.route.ts create mode 100644 src/workers/emailWorker.ts create mode 100644 src/workers/imageProcessingWorker.ts create mode 100644 src/workers/index.ts diff --git a/package-lock.json b/package-lock.json index 9110403..4fa6a74 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "license": "ISC", "dependencies": { + "bullmq": "^3.13.4", "env-cmd": "^10.1.0", "express": "^4.18.2", "joi": "^17.9.2", @@ -874,6 +875,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1383,6 +1389,78 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2444,8 +2522,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/base64-js": { "version": "1.5.1", @@ -2630,6 +2707,77 @@ "node": ">=10" } }, + "node_modules/bullmq": { + "version": "3.13.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.13.4.tgz", + "integrity": "sha512-3B16ZbOQSr9VoWpUXhoLB4khgxAbqwIiboJItA5rU2CEkLRe613w8yGxrivNdjJ6q8RcZtS+QvK0WHqCsuQYTQ==", + "dependencies": { + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/bullmq/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/bullmq/node_modules/semver": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", + "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/bullmq/node_modules/tslib": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.2.tgz", + "integrity": "sha512-5svOrSA2w3iGFDs1HibEVBGbDrAY82bFQ3HZ3ixB+88nsbsWQoKqDRb5UBYAUPEzbBn6dAp5gRNXglySbx1MlA==" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2796,6 +2944,14 @@ "node": ">=12" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -2912,6 +3068,17 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2978,6 +3145,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4024,8 +4199,7 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/fsevents": { "version": "2.3.2", @@ -4450,7 +4624,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -4474,6 +4647,50 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ip": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", @@ -5615,6 +5832,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -5631,7 +5863,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -5639,6 +5870,14 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", + "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==", + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -5904,6 +6143,35 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/msgpackr": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz", + "integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -5924,6 +6192,17 @@ "node": ">= 0.6" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -6080,7 +6359,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -6532,6 +6810,25 @@ "node": ">=8.10.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regexp.prototype.flags": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", @@ -6934,6 +7231,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -7563,6 +7865,14 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -7728,8 +8038,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { "version": "4.0.2", @@ -7756,8 +8065,7 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { "version": "17.7.2", diff --git a/package.json b/package.json index e7bada1..ca7dcb9 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ }, "homepage": "https://github.com/TogetherCrew/typescript-service#readme", "dependencies": { + "bullmq": "^3.13.4", "env-cmd": "^10.1.0", "express": "^4.18.2", "joi": "^17.9.2", diff --git a/src/app.ts b/src/app.ts index 60b007f..c46c8b8 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,9 +1,9 @@ import express, { type Express, type Request, type Response } from "express"; -import postsRoute from "./routes/posts.route"; +import routes from "./routes"; const app: Express = express(); - -app.use("/posts", postsRoute); +app.use(express.json()) +app.use(routes) app.get("/", (req: Request, res: Response) => { res.send("Express + TypeScript Server"); diff --git a/src/controllers/jobs/createJob.controller.ts b/src/controllers/jobs/createJob.controller.ts new file mode 100644 index 0000000..002ee3a --- /dev/null +++ b/src/controllers/jobs/createJob.controller.ts @@ -0,0 +1,27 @@ +import { type Request, type Response } from "express"; +import { type Queue } from "bullmq"; +import { queueByName } from "../../queues"; + +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +const createJob = async function (req: Request, res: Response) { + try { + const { type, data } = req.body; // Assuming you send the job type and data in the request body + + let queue: Queue + try { + queue = queueByName(type) + } catch (error) { + console.error(error) + return res.status(400).json({ error }); + } + + const job = await queue.add(type, data); // Add the job to the appropriate queue + + return res.json({ jobId: job.id }); + } catch (error) { + console.error('Error creating job:', error); + return res.status(500).json({ error: 'Failed to create job' }); + } +} + +export default createJob \ No newline at end of file diff --git a/src/controllers/jobs/getJob.controller.ts b/src/controllers/jobs/getJob.controller.ts new file mode 100644 index 0000000..2d8755a --- /dev/null +++ b/src/controllers/jobs/getJob.controller.ts @@ -0,0 +1,38 @@ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +import { type Request, type Response } from "express"; +import { Job, type Queue } from "bullmq"; +import { queueByName } from "../../queues"; + +const getJob = async function (req: Request, res: Response) { + try { + const { type, jobId } = req.params; // Assuming the jobId is passed as a route parameter + + let queue: Queue + try { + queue = queueByName(type) + } catch (error) { + console.error(error) + return res.status(400).json({ error }); + } + + // Fetch the job by its id + const job = await Job.fromId(queue, jobId); + + if (job == null) { + return res.status(404).json({ error: 'Job not found' }); + } + + return res.json({ + id: job.id, + name: job.name, + status: await job.getState(), + progress: job.progress, + data: job.data, + }); + } catch (error) { + console.error('Error getting job status:', error); + return res.status(500).json({ error: 'Failed to get job status' }); + } +} + +export default getJob \ No newline at end of file diff --git a/src/controllers/jobs/index.ts b/src/controllers/jobs/index.ts new file mode 100644 index 0000000..eecf4f7 --- /dev/null +++ b/src/controllers/jobs/index.ts @@ -0,0 +1,7 @@ +import createJob from "./createJob.controller"; +import getJob from "./getJob.controller"; + +export { + createJob, + getJob +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index baf84b8..2142229 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import mongoose from "mongoose"; import app from "./app"; import { env } from "./config"; +import './workers' mongoose.set("strictQuery", true); diff --git a/src/queues/connection.ts b/src/queues/connection.ts new file mode 100644 index 0000000..2ae423b --- /dev/null +++ b/src/queues/connection.ts @@ -0,0 +1,9 @@ +import { type RedisOptions } from "bullmq"; +import { env } from "../config"; + +const connection: RedisOptions = { + host: env.REDIS_QUEUE_HOST, + port: env.REDIS_QUEUE_PORT +} + +export default connection \ No newline at end of file diff --git a/src/queues/email.queue.ts b/src/queues/email.queue.ts new file mode 100644 index 0000000..505da4b --- /dev/null +++ b/src/queues/email.queue.ts @@ -0,0 +1,6 @@ +import { Queue } from "bullmq"; +import connection from "./connection"; + +const emailQueue = new Queue('email', { connection }) + +export default emailQueue; \ No newline at end of file diff --git a/src/queues/imageProcessing.queue.ts b/src/queues/imageProcessing.queue.ts new file mode 100644 index 0000000..7d44a5a --- /dev/null +++ b/src/queues/imageProcessing.queue.ts @@ -0,0 +1,6 @@ +import { Queue } from "bullmq"; +import connection from "./connection"; + +const imageProcessingQueue = new Queue('imageProcessing', { connection }) + +export default imageProcessingQueue; \ No newline at end of file diff --git a/src/queues/index.ts b/src/queues/index.ts new file mode 100644 index 0000000..4de6dfd --- /dev/null +++ b/src/queues/index.ts @@ -0,0 +1,9 @@ +import emailQueue from "./email.queue"; +import imageProcessingQueue from "./imageProcessing.queue" +import queueByName from "./queueByName"; + +export { + emailQueue, + imageProcessingQueue, + queueByName +} \ No newline at end of file diff --git a/src/queues/queueByName.ts b/src/queues/queueByName.ts new file mode 100644 index 0000000..3a8a2de --- /dev/null +++ b/src/queues/queueByName.ts @@ -0,0 +1,16 @@ +import { type Queue } from "bullmq"; +import emailQueue from "./email.queue"; +import imageProcessingQueue from "./imageProcessing.queue"; + +const queueByName = (name: string): Queue => { + switch (name) { + case emailQueue.name: + return emailQueue; + case imageProcessingQueue.name: + return imageProcessingQueue; + default: + throw new Error(`No Queue called ${name}`) + } +} + +export default queueByName \ No newline at end of file diff --git a/src/routes/index.ts b/src/routes/index.ts new file mode 100644 index 0000000..8f81d99 --- /dev/null +++ b/src/routes/index.ts @@ -0,0 +1,10 @@ +import express from "express"; +import jobsRoute from "./jobs.route" +import postsRoute from "./posts.route" + +const router = express.Router(); + +router.use("/jobs", jobsRoute) +router.use("/posts", postsRoute) + +export default router \ No newline at end of file diff --git a/src/routes/jobs.route.ts b/src/routes/jobs.route.ts new file mode 100644 index 0000000..90a806f --- /dev/null +++ b/src/routes/jobs.route.ts @@ -0,0 +1,10 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import express from "express"; +import { createJob, getJob } from "../controllers/jobs"; + +const router = express.Router(); + +router.post("/", createJob); +router.get("/:type/:jobId", getJob) + +export default router; diff --git a/src/workers/emailWorker.ts b/src/workers/emailWorker.ts new file mode 100644 index 0000000..26b3df3 --- /dev/null +++ b/src/workers/emailWorker.ts @@ -0,0 +1,42 @@ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +import { Worker } from 'bullmq'; +import { emailQueue } from '../queues'; // Assuming you have already defined the emailQueue +import connection from '../queues/connection'; + +// Create a new BullMQ worker instance for emailQueue +const emailWorker = new Worker(emailQueue.name, async (job) => { + // Define the processing logic for email jobs + const { to, subject, body } = job.data; // Assuming the job data contains the recipient, subject, and body of the email + + try { + // Perform email processing logic here + console.log(`Sending email to ${to}: ${subject}`); + console.log('Body:', body); + + // Simulating email sending time + await new Promise((resolve) => setTimeout(resolve, 2000)); + + console.log('Email sent successfully'); + } catch (error) { + console.error('Error processing email:', error); + throw new Error('Failed to process email'); // Throw an error if email processing fails + } +}, { connection }); + +emailWorker.on('failed', (job, err) => { + if (job !== undefined) { + console.error(`Job ${job.id} failed with error:`, err); + } else { + console.error('The job was undefined.') + } +}); + +emailWorker.on('completed', (job) => { + console.log(`Job ${job.id} completed successfully`); +}); + +emailWorker.on('error', (error) => { + console.error('Worker error:', error); +}); + +export default emailWorker \ No newline at end of file diff --git a/src/workers/imageProcessingWorker.ts b/src/workers/imageProcessingWorker.ts new file mode 100644 index 0000000..48b171c --- /dev/null +++ b/src/workers/imageProcessingWorker.ts @@ -0,0 +1,41 @@ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +import { Worker } from 'bullmq'; +import { imageProcessingQueue } from '../queues'; // Assuming you have already defined the imageProcessingQueue +import connection from '../queues/connection'; + +// Create a new BullMQ worker instance for imageProcessingQueue +const imageProcessingWorker = new Worker(imageProcessingQueue.name, async (job) => { + // Define the processing logic for image processing jobs + const { imageUrl } = job.data; // Assuming the job data contains the URL of the image to be processed + + try { + // Perform image processing logic here + console.log(`Processing image: ${imageUrl}`); + + // Simulating image processing time + await new Promise((resolve) => setTimeout(resolve, 3000)); + + console.log('Image processing completed'); + } catch (error) { + console.error('Error processing image:', error); + throw new Error('Failed to process image'); // Throw an error if image processing fails + } +}, { connection }); + +imageProcessingWorker.on('failed', (job, err) => { + if (job !== undefined) { + console.error(`Job ${job.id} failed with error:`, err); + } else { + console.error('The job was undefined.') + } +}); + +imageProcessingWorker.on('completed', (job) => { + console.log(`Job ${job.id} completed successfully`); +}); + +imageProcessingWorker.on('error', (error) => { + console.error('Worker error:', error); +}); + +export default imageProcessingWorker \ No newline at end of file diff --git a/src/workers/index.ts b/src/workers/index.ts new file mode 100644 index 0000000..080d666 --- /dev/null +++ b/src/workers/index.ts @@ -0,0 +1,7 @@ +import emailWorker from './emailWorker' +import imageProcessingWorker from './imageProcessingWorker' + +export { + emailWorker, + imageProcessingWorker +} \ No newline at end of file