fix: remove redis dependency (#1662)

.circleci/config.yml

@@ -6,7 +6,6 @@ executors:
   standard-node:
     docker:
       - image: "cimg/node:18.14.2"
-      - image: "circleci/redis:6.2.1-alpine"
       - image: "circleci/postgres:12.3-postgis"
         environment:
           POSTGRES_USER: bloom-ci
@@ -31,9 +30,6 @@ executors:
       DATABASE_URL: "postgres://bloom-ci@localhost:5432/bloom"
       # DB URL for the jest tests per ormconfig.test.ts
       TEST_DATABASE_URL: "postgres://bloom-ci@localhost:5432/bloom"
-      REDIS_TLS_URL: "rediss://localhost:6379/0"
-      REDIS_URL: "redis://localhost:6379/0"
-      REDIS_USE_TLS: "0"
       PARTNERS_PORTAL_URL: "http://localhost:3001"
 
 jobs:
@@ -83,9 +79,6 @@ jobs:
             DATABASE_URL: "postgres://bloom-ci@localhost:5432/bloom"
             # DB URL for the jest tests per ormconfig.test.ts
             TEST_DATABASE_URL: "postgres://bloom-ci@localhost:5432/bloom"
-            REDIS_TLS_URL: "rediss://localhost:6379/0"
-            REDIS_URL: "redis://localhost:6379/0"
-            REDIS_USE_TLS: "0"
             CLOUDINARY_SIGNED_PRESET: "fake_secret"
             CLOUDINARY_KEY: "fake_key"
             CLOUDINARY_CLOUD_NAME: "exygy"

.travis.yml

@@ -1,8 +1,6 @@
 language: node_js
 node_js:
   - 18
-services:
-  - redis-server
 before_install:
   - sudo sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf
   - sudo systemctl restart postgresql@12-main
@@ -61,6 +59,5 @@ env:
   global: PGPORT=5433
     PGUSER=travis
     TEST_DATABASE_URL=postgres://localhost:5433/bloom_test
-    REDIS_TLS_URL=redis://127.0.0.1:6379/0
     NEW_RELIC_ENABLED=false
     NEW_RELIC_LOG_ENABLED=false

backend/core/.env.template

@@ -2,9 +2,6 @@ PORT=3100
 NODE_ENV=development
 DATABASE_URL=postgres://localhost/bloom_detroit
 TEST_DATABASE_URL=postgres://localhost/bloom_test
-REDIS_TLS_URL=redis://127.0.0.1:6379/0
-REDIS_URL=redis://127.0.0.1:6379/0
-REDIS_USE_TLS=0
 THROTTLE_TTL=60
 THROTTLE_LIMIT=2
 EMAIL_API_KEY='SOME-LONG-SECRET-KEY'

backend/core/Dockerfile

@@ -30,6 +30,6 @@ COPY --from=development /usr/src/app/dist ./dist
 EXPOSE ${PORT}
 
 # If you attempt to run this by itself, you'll need to pass in:
-# PORT, DATABASE_URL, REDIS_URL, REDIS_TLS_URL
+# PORT, DATABASE_URL
 # Typically via: docker run --env-file=".env" -t my-app
 CMD yarn start

backend/core/README.md

@@ -10,27 +10,15 @@ OpenAPI (fka Swagger) documentation is automatically generated by the server at
 
 - Install Node.js 14.x `brew install node@14.`
 - Install Postgres 12 `brew install postgresql`
-- Install Redis `brew install redis`
 - Copy the `.env.template` within `backend/core` to `.env` and edit variables appropriate to your local environment. Ensure sure the Database URL and Test Database URL match your Postgres configuration.
 - Install dependencies `yarn install` within `backend/core`
 
-### Redis
-
-To start Redis:
-`redis-server`.
-
-To launch Redis as background service and restart at login:
-`brew services start redis`.
-
-Test if Redis is working:
-`redis-cli ping`
-
 ### Using Docker containers
 
-If you don't want to install Postgres and Redis on your local machine and instead want to use Docker containers, run:
+If you don't want to install Postgres on your local machine and instead want to use Docker containers, run:
 
 ```shell script
-docker-compose up redis postgres
+docker-compose up postgres
 ```
 
 All `psql` and `yarn` commands related to databases will then be connecting to the database in the docker container `postgres`.
@@ -108,10 +96,7 @@ translation pair:
 | PORT                | Port number the server will listen to for incoming connections                                                                             | 3100                            | number                        |
 | NODE_ENV            | Controls build optimization and enables some additional logging when set to `development`                                                  | development                     | "development" \| "production" |
 | DATABASE_URL        | Database connection                                                                                                                        | postgres://localhost/bloom      | string                        |
-| TEST_DATABASE_URL   | Test database connection                                                                                                                   | postgres://localhost/bloom_test | string                        | string |
-| REDIS_TLS_URL       | Secure Redis connection                                                                                                                    | rediss://127.0.0.1:6379/        | string                        |
-| REDIS_URL           | TCP Redis connection string                                                                                                                | redis://127.0.0.1:6379/0        | string                        |
-| REDIS_USE_TLS       | Flag controlling the use of TLS or unsecure transport for Redis                                                                            | 0                               | 0 \| 1                        |
+| TEST_DATABASE_URL   | Test database connection                                                                                                                   | postgres://localhost/bloom_test | string                        |
 | THROTTLE_TTL        | Rate limit TTL in seconds (currently used only for application submission endpoint)                                                        | 60                              | number                        |
 | THROTTLE_LIMIT      | Max number of operations in given time window THROTTLE_TTL after which HTTP 429 Too Many Requests will be returned by the server           | 2                               | number                        |
 | EMAIL_API_KEY       | Sendgrid API key (see [sendgrid docs for creating API keys](https://sendgrid.com/docs/ui/account-and-settings/api-keys/#managing-api-keys) | Available internally            | string                        |

backend/core/package.json

@@ -46,9 +46,7 @@
   "dependencies": {
     "@anchan828/nest-sendgrid": "^0.3.25",
     "@google-cloud/translate": "^6.2.6",
-    "@nestjs/bull": "^0.6.3",
     "@types/mapbox": "^1.6.42",
-    "bull": "^4.1.0",
     "@nestjs/cli": "^8.2.1",
     "@nestjs/common": "^8.3.1",
     "@nestjs/config": "^1.2.0",
@@ -65,7 +63,6 @@
     "async-retry": "^1.3.1",
     "axios": "0.21.2",
     "cache-manager": "^3.4.0",
-    "cache-manager-redis-store": "^2.0.0",
     "casbin": "5.13.0",
     "class-transformer": "0.3.1",
     "class-validator": "^0.12.2",
@@ -77,14 +74,12 @@
     "express": "^4.17.1",
     "fast-xml-parser": "^4.0.0-beta.2",
     "handlebars": "^4.7.6",
-    "ioredis": "^5.2.3",
     "joi": "^17.3.0",
     "jwt-simple": "^0.5.6",
     "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "mapbox": "^1.0.0-beta10",
     "nanoid": "^3.1.12",
-    "nestjs-throttler-storage-redis": "^0.1.18",
     "nestjs-twilio": "^2.1.0",
     "nestjs-typeorm-paginate": "^3.1.3",
     "newrelic": "7.5.1",
@@ -112,7 +107,6 @@
     "@nestjs/schematics": "^8.0.7",
     "@nestjs/testing": "^8.3.1",
     "@types/axios": "^0.14.0",
-    "@types/bull": "^3.15.5",
     "@types/cron": "^1.7.3",
     "@types/express": "^4.17.8",
     "@types/node": "^12.12.67",

backend/core/src/app.module.ts

@@ -23,8 +23,6 @@ import { AmiChartsModule } from "./ami-charts/ami-charts.module"
 import { ApplicationFlaggedSetsModule } from "./application-flagged-sets/application-flagged-sets.module"
 import * as bodyParser from "body-parser"
 import { ThrottlerModule } from "@nestjs/throttler"
-import { ThrottlerStorageRedisService } from "nestjs-throttler-storage-redis"
-import Redis from "ioredis"
 import { SharedModule } from "./shared/shared.module"
 import { ConfigModule, ConfigService } from "@nestjs/config"
 import { TranslationsModule } from "./translations/translations.module"
@@ -40,7 +38,6 @@ import { PaperApplicationsModule } from "./paper-applications/paper-applications
 import { SmsModule } from "./sms/sms.module"
 import { ScheduleModule } from "@nestjs/schedule"
 import { CronModule } from "./cron/cron.module"
-import { BullModule } from "@nestjs/bull"
 import { ProgramsModule } from "./program/programs.module"
 import { ActivityLogModule } from "./activity-log/activity-log.module"
 import { logger } from "./shared/middlewares/logger.middleware"
@@ -64,27 +61,6 @@ export function applicationSetup(app: INestApplication) {
 })
 export class AppModule {
   static register(dbOptions): DynamicModule {
-    /**
-     * DEV NOTE:
-     * This configuration is required due to issues with
-     * self signed certificates in Redis 6.
-     *
-     * { rejectUnauthorized: false } option is intentional and required
-     *
-     * Read more:
-     * https://help.heroku.com/HC0F8CUS/redis-connection-issues
-     * https://devcenter.heroku.com/articles/heroku-redis#ioredis-module
-     */
-    const redis =
-      "0" === process.env.REDIS_USE_TLS
-        ? new Redis(process.env.REDIS_URL, { connectTimeout: 60000 })
-        : new Redis(process.env.REDIS_TLS_URL, {
-            tls: {
-              rejectUnauthorized: false,
-            },
-            connectTimeout: 60000,
-          })
-
     return {
       module: AppModule,
       imports: [
@@ -94,9 +70,6 @@ export class AppModule {
         ApplicationsModule,
         AssetsModule,
         AuthModule,
-        BullModule.forRoot({
-          redis: redis.options,
-        }),
         JurisdictionsModule,
         ListingsModule,
         CronModule,
@@ -121,7 +94,6 @@ export class AppModule {
           useFactory: (config: ConfigService) => ({
             ttl: config.get("THROTTLE_TTL"),
             limit: config.get("THROTTLE_LIMIT"),
-            storage: new ThrottlerStorageRedisService(redis),
           }),
         }),
         UnitsModule,

backend/core/src/listings/listings.module.ts

@@ -3,7 +3,6 @@ import { TypeOrmModule } from "@nestjs/typeorm"
 import { ListingsService } from "./listings.service"
 import { ListingsController } from "./listings.controller"
 import { Listing } from "./entities/listing.entity"
-import { ListingsNotificationsConsumer } from "./listings-notifications"
 import { Unit } from "../units/entities/unit.entity"
 import { Preference } from "../preferences/entities/preference.entity"
 import { AuthModule } from "../auth/auth.module"
@@ -41,14 +40,8 @@ import { CsvBuilder } from "../../src/applications/services/csv-builder.service"
     SmsModule,
     ActivityLogModule,
   ],
-  providers: [
-    ListingsService,
-    ListingsNotificationsConsumer,
-    CsvBuilder,
-    ListingsCsvExporterService,
-  ],
+  providers: [ListingsService, CsvBuilder, ListingsCsvExporterService],
   exports: [ListingsService],
   controllers: [ListingsController],
 })
-// We have to manually disconnect from redis on app close
 export class ListingsModule {}

backend/core/src/listings/tests/listings.service.spec.ts

@@ -14,7 +14,6 @@ import { ContextIdFactory } from "@nestjs/core"
 import { UnitGroup } from "../../units-summary/entities/unit-group.entity"
 import { UnitType } from "../../unit-types/entities/unit-type.entity"
 import { Program } from "../../program/entities/program.entity"
-import { BullModule, getQueueToken } from "@nestjs/bull"
 import { User } from "../../../src/auth/entities/user.entity"
 
 // Cypress brings in Chai types for the global expect, but we want to use jest
@@ -108,8 +107,6 @@ const mockListingsRepo = {
   save: jest.fn(),
 }
 
-const mockListingsNotificationsQueue = { add: jest.fn() }
-
 describe("ListingsService", () => {
   beforeEach(async () => {
     process.env.APP_SECRET = "SECRET"
@@ -147,11 +144,7 @@ describe("ListingsService", () => {
         },
         { provide: getRepositoryToken(User), useValue: jest.fn() },
       ],
-      imports: [BullModule.registerQueue({ name: "listings-notifications" })],
-    })
-      .overrideProvider(getQueueToken("listings-notifications"))
-      .useValue(mockListingsNotificationsQueue)
-      .compile()
+    }).compile()
 
     const contextId = ContextIdFactory.create()
     jest.spyOn(ContextIdFactory, "getByRequest").mockImplementation(() => contextId)

backend/core/src/shared/shared.module.ts

@@ -12,8 +12,6 @@ import Joi from "joi"
           .default("development"),
         EMAIL_API_KEY: Joi.string().required(),
         DATABASE_URL: Joi.string().required(),
-        REDIS_TLS_URL: Joi.string().required(),
-        REDIS_USE_TLS: Joi.number().required(),
         THROTTLE_TTL: Joi.number().default(1),
         THROTTLE_LIMIT: Joi.number().default(100),
         APP_SECRET: Joi.string().required().min(16),

docker-compose.yml

@@ -77,21 +77,11 @@ services:
     environment:
       POSTGRES_USER: "postgres"
       DATABASE_URL: "postgres://postgres:5432/bloom"
-      REDIS_TLS_URL: "redis://redis:6379/0"
-      REDIS_URL: "redis://redis:6379/0"
       PGUSER: "postgres"
       PGPASSWORD: "postgres"
       PGDATABASE: "bloom"
     depends_on:
-      - redis
       - postgres
-  redis:
-    container_name: redis
-    image: redis:latest
-    ports:
-      - "6379:6379"
-    networks:
-      - backend
   postgres:
     container_name: postgres
     image: postgres:13

docs/DeployServicesHeroku.md

@@ -5,7 +5,6 @@ Bloom is designed to use a set of independently run services that provide the da
 ## Resources
 
 - [Heroku Postgres](https://www.heroku.com/postgres)
-- [Heroku Redis](https://www.heroku.com/redis)
 
 ## Heroku Buildpacks
 
@@ -38,9 +37,3 @@ DATABASE_URL=
 EMAIL_API_KEY='SENDGRID-API-KEY'
 
 PARTNERS_BASE_URL='PARTNER-PORTAL-URL'
-
-REDIS_TLS_URL=
-
-REDIS_URL=
-
-REDIS_USE_TLS=1