Merge pull request #2253 from HHS/jp/3112/rm-elasticsearch

[TTAHUB-3112] Remove ElasticSearch

.circleci/config.yml

@@ -19,27 +19,6 @@ executors:
  docker-python-executor:
  docker:
  - image: cimg/python:3.9.19
- docker-postgres-elasticsearch-executor:
- docker:
- - image: cimg/node:18.20.3-browsers
- environment:
- DATABASE_URL: postgresql://postgres@localhost/ttasmarthub
- - image: cimg/postgres:15.6
- environment:
- POSTGRES_USER: postgres
- POSTGRES_PASSWORD: secretpass
- POSTGRES_DB: ttasmarthub
- - image: opensearchproject/opensearch:1.3.2
- name: opensearch
- environment:
- transport.host: 0.0.0.0
- network.host: 0.0.0.0
- http.port: 9200
- cluster.name: opensearch-cluster
- node.name: opensearch-node1
- discovery.type: single-node
- DISABLE_SECURITY_PLUGIN: true
- ES_JAVA_OPTS: "-Xms256m -Xmx256m"
  machine-executor:
  machine:
  image: ubuntu-2204:current
@@ -427,7 +406,7 @@ parameters:
  default: "al-ttahub-2939-add-fei-root-cause-to-goal-card"
  type: string
  sandbox_git_branch: # change to feature branch to test deployment
- default: "jp/3005/ipd-courses-widget"
+ default: "jp/3112/rm-elasticsearch"
  type: string
  prod_new_relic_app_id:
  default: "877570491"
@@ -536,11 +515,8 @@ jobs:
  - similarity_api
  resource_class: large
  test_backend:
- executor: docker-postgres-elasticsearch-executor
+ executor: docker-postgres-executor
  environment:
- AWS_ELASTICSEARCH_ENDPOINT: http://opensearch:9200
- AWS_ELASTICSEARCH_ACCESS_KEY: admin
- AWS_ELASTICSEARCH_SECRET_KEY: admin
  SFTP_EXPOSED_PORT: 2222
  steps:
  - attach_workspace:
@@ -636,11 +612,7 @@ jobs:
  path: frontend/coverage/
  resource_class: large
  test_e2e:
- executor: docker-postgres-elasticsearch-executor
- environment:
- AWS_ELASTICSEARCH_ENDPOINT: http://opensearch:9200
- AWS_ELASTICSEARCH_ACCESS_KEY: admin
- AWS_ELASTICSEARCH_SECRET_KEY: admin
+ executor: docker-postgres-executor
  steps:
  - attach_workspace:
  at: .
@@ -677,11 +649,7 @@ jobs:
  path: playwright/e2e
  resource_class: large
  test_api:
- executor: docker-postgres-elasticsearch-executor
- environment:
- AWS_ELASTICSEARCH_ENDPOINT: http://opensearch:9200
- AWS_ELASTICSEARCH_ACCESS_KEY: admin
- AWS_ELASTICSEARCH_SECRET_KEY: admin
+ executor: docker-postgres-executor
  steps:
  - attach_workspace:
  at: .
@@ -718,11 +686,7 @@ jobs:
  path: playwright/api
  resource_class: large
  test_utils:
- executor: docker-postgres-elasticsearch-executor
- environment:
- AWS_ELASTICSEARCH_ENDPOINT: http://opensearch:9200
- AWS_ELASTICSEARCH_ACCESS_KEY: admin
- AWS_ELASTICSEARCH_SECRET_KEY: admin
+ executor: docker-postgres-executor
  steps:
  - attach_workspace:
  at: .

.env.example

@@ -52,11 +52,6 @@ CLAMAV_ENDPOINT=https://clamav-rest:9443
 # CLAMAV_ENDPOINT=http://localhost:8081
 REDIS_PASS=SUPERSECUREPASSWORD
 
-# Vars for AWS OpenSearch (ES) local docker image
-AWS_ELASTICSEARCH_ENDPOINT=http://localhost:9200
-AWS_ELASTICSEARCH_ACCESS_KEY=admin
-AWS_ELASTICSEARCH_SECRET_KEY=admin
-
 # Email Address that notifications should come from
 FROM_EMAIL_ADDRESS=[email protected]
 

README.md

@@ -47,11 +47,6 @@ The frontend [proxies requests](https://create-react-app.dev/docs/proxying-api-r
 
 Api documentation uses [Redoc](https://github.com/Redocly/redoc) to serve documentation files. These files can be found in the `docs/openapi` folder. Api documentation should be split into separate files when appropriate to prevent huge hard to grasp yaml files.
 
-We use an AWS OpenSearch docker image (Elasticsearch fork) and require that the following variables get added to the env file.
-* `AWS_ELASTICSEARCH_ENDPOINT=http://opensearch-node1:9200`
-* `AWS_ELASTICSEARCH_ACCESS_KEY=admin`
-* `AWS_ELASTICSEARCH_SECRET_KEY=admin`
-
 #### Apple Silicon & Chromium
 On a Mac with Apple Silicon, puppeteer install fails with the message:
 ```"The chromium binary is not available for arm64"```
@@ -73,11 +68,6 @@ You can also run build commands directly on your host (without docker). Make sur
 
 You must also install and run minio locally to use the file upload functionality. Please comment out `S3_ENDPOINT=http://minio:9000` and uncomment `S3_ENDPOINT=http://localhost:9000` in your .env file.
 
-We use an AWS OpensSearch docker image (Elasticsearch fork) and require that the following variables get added to the env file.
-* `AWS_ELASTICSEARCH_ENDPOINT=http://localhost:9200`
-* `AWS_ELASTICSEARCH_ACCESS_KEY=admin`
-* `AWS_ELASTICSEARCH_SECRET_KEY=admin`
-
 #### Precommit hooks
 
 Our CI will fail if code is committed that doesn't pass our linter (eslint). This repository contains a pre-commit hook that runs eslint's built in "fix" command on all staged javascript files so that any autofixable errors will be fixed. The precommit hook, in .gihooks/pre-commit, also contains code to auto-format our terraform files, which you can read more about [here](https://github.com/adhocteam/Head-Start-TTADP/tree/main/terraform#set-up).
@@ -97,12 +87,12 @@ If you are already using git hooks, add the .githooks/pre-commit contents to you
 ### Building Tests
 
 #### Helpful notes on writing (backend) tests
-It's important that our tests fully clean up after themselves if they interact with the database. This way, tests do not conflict when run on the CI and remain as deterministic as possible.The best way to do this is to run them locally in an isolated environment and confirm that they are sanitary. 
+It's important that our tests fully clean up after themselves if they interact with the database. This way, tests do not conflict when run on the CI and remain as deterministic as possible.The best way to do this is to run them locally in an isolated environment and confirm that they are sanitary.
 
 With that in mind, there a few "gotchas" to remember to help write sanitary tests.
 - ```Grant.destroy``` needs to run with ```individualHooks: true``` or the related GrantNumberLink model prevents delete.
 - When you call ```Model.destroy``` you should be adding ```individualHooks: true``` to the Sequelize options. Often this is required for proper cleanup. There may be times when this is undesirable; this should be indicated with a comment.
-- Be aware of paranoid models. For those models: force: true gets around the soft delete. If they are already soft-deleted though, you need to remove the default scopes paranoid: true does it, as well as Model.unscoped() 
+- Be aware of paranoid models. For those models: force: true gets around the soft delete. If they are already soft-deleted though, you need to remove the default scopes paranoid: true does it, as well as Model.unscoped()
 - There are excellent helpers for creating and destroying common Model mocks in ```testUtils.js```. Be aware that they take a scorched earth approach to cleanup. For example, when debugging a flaky test, it was discovered that ```destroyReport``` was removing a commonly used region.
 - The next section details additional tools, found in `src/lib/programmaticTransaction.ts`, which make maintaining a clean database state when writing tests a breeze.
 
@@ -627,4 +617,3 @@ ex:
 [backend]:http://localhost:8080
 [API documentation]:http://localhost:5003
 [minio]:http://localhost:3000
-[elasticsearch]:http://localhost:9200

docker-compose.yml

@@ -59,20 +59,6 @@ services:
  ports:
  - "1025:1025"
  - "1080:1080"
- opensearch-node1:
- image: opensearchproject/opensearch:1.3.2
- env_file: .env
- container_name: opensearch-node1
- environment:
- - cluster.name=opensearch-cluster
- - node.name=opensearch-node1
- - "DISABLE_SECURITY_PLUGIN=true" # disables security plugin entirely in OpenSearch by setting plugins.security.disabled: true in opensearch.yml
- - "discovery.type=single-node" # disables bootstrap checks that are enabled when network.host is set to a non-loopback address
- volumes:
- - opensearch-data1:/usr/share/opensearch/data
- ports:
- - 9200:9200
- - 9600:9600 # required for Performance Analyzer
  testingonly:
  build:
  context: .
@@ -97,4 +83,3 @@ services:
 volumes:
  dbdata: {}
  minio-data: {}
- opensearch-data1: {}

docs/boundary_diagram.md

@@ -30,7 +30,6 @@ Boundary(aws, "AWS GovCloud") {
  ContainerDb(www_db, "PostgreSQL Database", "AWS RDS", "Contains content and configuration for the TTA Hub")
  ContainerDb(www_s3, "AWS S3 bucket", "AWS S3", "Stores static file assets")
  ContainerDb(www_redis, "Redis Database", "AWS Elasticache", "Queue of background jobs to work on")
- Container(Elasticache, "Elasticache", "AWS Elasticache", "Elasticache for search results")
  }
  }
  }

docs/openapi/paths/index.yaml

@@ -64,8 +64,6 @@
  $ref: './widgets/reasonList.yaml'
 '/widgets/topicAndFrequencyGraph':
  $ref: './widgets/topicAndFrequencyGraph.yaml'
-'/search':
- $ref: './search/search.yaml'
 '/settings':
  $ref: './settings/index.yaml'
 '/settings/email':

frontend/package.json

@@ -9,8 +9,8 @@
  "@fortawesome/free-solid-svg-icons": "^6.4.2",
  "@fortawesome/react-fontawesome": "^0.2.0",
  "@hookform/error-message": "^0.0.5",
- "@silevis/reactgrid": "3.1",
  "@react-hook/resize-observer": "^1.2.6",
+ "@silevis/reactgrid": "3.1",
  "@trussworks/react-uswds": "4.1.1",
  "@ttahub/common": "^2.1.5",
  "@use-it/interval": "^1.0.0",
@@ -213,4 +213,4 @@
  }
  }
  }
-}
+}

frontend/yarn.lock

@@ -2374,9 +2374,9 @@
  integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==
 
 "@ttahub/common@^2.1.5":
- version "2.1.5"
- resolved "https://registry.yarnpkg.com/@ttahub/common/-/common-2.1.5.tgz#1e9550c507a6e4422eae72a923eed7c18613e827"
- integrity sha512-fy2jgE7xdaifZsxF+uiqwGpsxa0UchjPUNI2BI84Rw72FiTAglpkfn4wy4yO82NW7PyIa9CbICa4sP+XcKwKMg==
+ version "2.1.6"
+ resolved "https://registry.yarnpkg.com/@ttahub/common/-/common-2.1.6.tgz#259d98201d394eafce7686f8334768091af4655d"
+ integrity sha512-/X/suR8B5aKYuVXXRHa1gjBTMzzz7vyXDCwATkZ4McQhoil8dtzndYgACDFY5bC+ZsEIfqiTcDQ+Ssle1N9mbA==
 
 "@turf/area@^6.4.0":
  version "6.5.0"

manifest.yml

@@ -42,7 +42,6 @@ applications:
  - ttahub-((env))
  - ttahub-redis-((env))
  - ttahub-document-upload-((env))
- - ttahub-elasticsearch-((env))
  processes:
  - type: web
  instances: ((web_instances))

package.json

@@ -96,8 +96,6 @@
  "restoreTopics:local": "./node_modules/.bin/babel-node ./src/tools/restoreTopicsCLI.js",
  "coverage:backend": "./bin/build-coverage-report",
  "docker:coverage:backend": "docker compose run --rm backend chmod 744 ./bin/build-coverage-report && ./bin/build-coverage-report",
- "createAwsElasticSearchIndexes": "node ./build/server/src/tools/createAwsElasticSearchIndexesCli.js",
- "createAwsElasticSearchIndexes:local": "./node_modules/.bin/babel-node ./src/tools/createAwsElasticSearchIndexesCli.js",
  "populateLegacyResourceTitles": "node ./build/server/src/tools/populateLegacyResourceTitlesCli.js",
  "populateLegacyResourceTitles:local": "./node_modules/.bin/babel-node ./src/tools/populateLegacyResourceTitlesCli.js",
  "updateCompletedEventReportPilots": "node ./build/server/src/tools/updateCompletedEventReportPilotsCLI.js",
@@ -192,7 +190,6 @@
  "<rootDir>/src/app.js",
  "<rootDir>/src/tools/*CLI.js",
  "<rootDir>/src/tools/*Cli.js",
- "<rootDir>/src/tools/createAwsElasticSearchIndexesCli.js",
  "<rootDir>/src/tools/populateLegacyResourceTitlesCLI.js",
  "<rootDir>/src/tools/importTTAPlanGoals.js"
  ],
@@ -215,7 +212,6 @@
  "<rootDir>/src/lib/cron.js",
  "<rootDir>/src/tools/*CLI.js",
  "<rootDir>/src/tools/*Cli.js",
- "<rootDir>/src/tools/createAwsElasticSearchIndexesCli.js",
  "<rootDir>/src/tools/populateLegacyResourceTitlesCli.js",
  "<rootDir>/src/tools/importTTAPlanGoals.js"
  ],
@@ -296,11 +292,9 @@
  },
  "dependencies": {
  "@babel/runtime": "^7.12.1",
- "@elastic/elasticsearch": "^8.2.0-patch.1",
- "@elastic/elasticsearch-mock": "^2.0.0",
  "@faker-js/faker": "^6.0.0",
  "@opensearch-project/opensearch": "^1.1.0",
- "@ttahub/common": "^2.1.5",
+ "@ttahub/common": "^2.1.6",
  "adm-zip": "^0.5.1",
  "aws-sdk": "^2.826.0",
  "aws4": "^1.11.0",

packages/common/README.md

@@ -21,7 +21,11 @@ Note: On Windows you will need to use `yarn add @ttahub/[email protected]` to update
 
 ## Versions
 
-## 1.4.0 
+## 2.1.6
+
+Remove constants related to ElasticSearch.
+
+## 1.4.0
 Add SUPPORT_TYPE
 
 ### 1.1.9/1.2.0
@@ -40,7 +44,7 @@ Distinguish between training report url params and training report statuses
 Add "event target populations"
 
 ### 1.1.2
-#### 1.1.1 
+#### 1.1.1
 - Add COLLABORATOR_TRAINING_REPORTS permissions
 
 

packages/common/package.json

@@ -1,6 +1,6 @@
 {
  "name": "@ttahub/common",
- "version": "2.1.5",
+ "version": "2.1.6",
  "description": "The purpose of this package is to reduce code duplication between the frontend and backend projects.",
  "main": "src/index.js",
  "author": "",

src/constants.js

@@ -149,12 +149,6 @@ const EMAIL_ACTIONS = {
  TRAINING_REPORT_EVENT_COMPLETED: 'trainingReportEventCompleted',
 };
 
-const AWS_ELASTICSEARCH_ACTIONS = {
- ADD_INDEX_DOCUMENT: 'addIndexDocument',
- UPDATE_INDEX_DOCUMENT: 'updateIndexDocument',
- DELETE_INDEX_DOCUMENT: 'deleteIndexDocument',
-};
-
 const S3_ACTIONS = {
  DELETE_FILE: 'deleteFile',
 };
@@ -220,10 +214,6 @@ const RESOURCE_DOMAIN = {
  ECLKC: 'eclkc.ohs.acf.hhs.gov',
 };
 
-const AWS_ELASTIC_SEARCH_INDEXES = {
- ACTIVITY_REPORTS: 'activityreports',
-};
-
 const GRANT_INACTIVATION_REASONS = {
  REPLACED: 'Replaced',
  TERMINATED: 'Terminated',
@@ -285,14 +275,12 @@ module.exports = {
  RESOURCE_ACTIONS,
  USER_SETTINGS,
  EMAIL_ACTIONS,
- AWS_ELASTICSEARCH_ACTIONS,
  S3_ACTIONS,
  EMAIL_DIGEST_FREQ,
  DIGEST_SUBJECT_FREQ,
  PROMPT_FIELD_TYPE,
  SOURCE_FIELD,
  RESOURCE_DOMAIN,
- AWS_ELASTIC_SEARCH_INDEXES,
  GRANT_INACTIVATION_REASONS,
  MAINTENANCE_TYPE,
  MAINTENANCE_CATEGORY,