From f3f7df486e0feabdd672e3d7776c7dab49cde90d Mon Sep 17 00:00:00 2001 From: Yaroslav Grishajev Date: Mon, 16 Sep 2024 12:36:17 +0200 Subject: [PATCH] feat(env): unify app configs for api and indexer, update doc refs #313 --- README.md | 1 + apps/api/README.md | 17 ----- apps/api/package.json | 1 + apps/api/src/console.ts | 2 +- apps/api/src/dotenv.ts | 23 ------- apps/api/src/index.ts | 2 +- apps/deploy-web/README.md | 33 ---------- apps/indexer/env/.env | 0 apps/indexer/env/.env.mainnet | 1 + apps/indexer/env/.env.production | 2 + apps/indexer/env/.env.sample | 7 ++ apps/indexer/env/.env.sandbox | 1 + apps/indexer/env/.env.testnet | 1 + apps/indexer/package.json | 3 +- apps/indexer/src/index.ts | 2 + doc/apps-configuration.md | 106 +++++++++++++++++++++++++++++++ package-lock.json | 2 + 17 files changed, 128 insertions(+), 76 deletions(-) delete mode 100644 apps/api/src/dotenv.ts delete mode 100644 apps/indexer/env/.env create mode 100644 apps/indexer/env/.env.mainnet create mode 100644 apps/indexer/env/.env.production create mode 100644 apps/indexer/env/.env.sample create mode 100644 apps/indexer/env/.env.sandbox create mode 100644 apps/indexer/env/.env.testnet create mode 100644 doc/apps-configuration.md diff --git a/README.md b/README.md index 62ffa3639..5759ac5dc 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - [Quick Start](#quick-start) +- [Apps Configuration](./doc/apps-configuration.md) - [Services](#services) - [Monitoring](#monitoring) - [Example SQL Queries](#example-sql-queries) diff --git a/apps/api/README.md b/apps/api/README.md index 29b490fce..849ea525a 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -13,23 +13,6 @@ You can make sure the api is working by accessing the status endpoint: `http://localhost:3080/status` -## Environment Variables - -This app utilizes `.env*` files to manage environment variables. The list of environment variables can be found in the `env/.env.sample` file. These files are included in version control and should only contain non-sensitive values. Sensitive values are provided by the deployment system. - -### Important Notes: -- **Sensitive Values**: The only env file that's ignored by Git is `env/.env.local`, which is intended for sensitive values used in development. -- **Loading Order**: Environment files are loaded in a specific order, depending on two environment variables: `DEPLOYMENT_ENV` and `NETWORK`. - -### Loading Order: -1. `env/.env.local` - Contains sensitive values for development. -2. `env/.env` - Default values applicable to all environments. -3. `env/.env.${DEPLOYMENT_ENV}` - Values specific to the deployment environment. -4. `env/.env.${NETWORK}` - Values specific to the network. - -### Additional Details: -- **Variable Precedence**: If a variable is already set in the environment, it will not be overridden by values in the `.env*` files. This behavior is critical when adjusting the loading order of these files. - ## Testing Project is configured to use [Jest](https://jestjs.io/) for testing. It is intended to be covered with unit and functional tests where applicable. diff --git a/apps/api/package.json b/apps/api/package.json index 56ce881a2..f6e2f54bf 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -34,6 +34,7 @@ "@akashnetwork/akash-api": "^1.3.0", "@akashnetwork/akashjs": "^0.10.0", "@akashnetwork/database": "*", + "@akashnetwork/env-loader": "*", "@akashnetwork/http-sdk": "*", "@casl/ability": "^6.7.1", "@chain-registry/assets": "^1.64.79", diff --git a/apps/api/src/console.ts b/apps/api/src/console.ts index 979d364db..a4b7e5873 100644 --- a/apps/api/src/console.ts +++ b/apps/api/src/console.ts @@ -1,5 +1,5 @@ import "reflect-metadata"; -import "./dotenv"; +import "@akashnetwork/env-loader"; import "./open-telemetry"; import { context, trace } from "@opentelemetry/api"; diff --git a/apps/api/src/dotenv.ts b/apps/api/src/dotenv.ts deleted file mode 100644 index 0d6a34982..000000000 --- a/apps/api/src/dotenv.ts +++ /dev/null @@ -1,23 +0,0 @@ -import dotenv from "@dotenvx/dotenvx"; -import fs from "fs"; -import pino from "pino"; - -const logger = pino().child({ context: "ENV" }); - -const config = (path: string) => { - if (fs.existsSync(path)) { - dotenv.config({ path }); - logger.info(`Loaded ${path}`); - } -}; -config("env/.env.local"); -config("env/.env"); - -const deploymentEnv = process.env.DEPLOYMENT_ENV; - -if (deploymentEnv && deploymentEnv !== "local") { - config(`env/.env.${deploymentEnv}`); -} - -const network = process.env.NETWORK || "mainnet"; -config(`env/.env.${network}`); diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5885ef7ea..c058d6fbc 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,6 +1,6 @@ import "reflect-metadata"; +import "@akashnetwork/env-loader"; import "./open-telemetry"; -import "./dotenv"; async function bootstrap() { /* eslint-disable @typescript-eslint/no-var-requires */ diff --git a/apps/deploy-web/README.md b/apps/deploy-web/README.md index a44b16280..986625000 100644 --- a/apps/deploy-web/README.md +++ b/apps/deploy-web/README.md @@ -11,36 +11,3 @@ 4. Start the app with `npm run dev`. The website should be accessible: [http://localhost:3000/](http://localhost:3000/) - -## Environment Variables - -### Overview -Environment variables in this Next.js app follow the standard Next.js behavior, as documented in the [Next.js environment variables documentation](https://nextjs.org/docs/basic-features/environment-variables). This means that files like `.env.local` or `.env.production` will be automatically loaded based on the environment in which the app is running. - -However, we have extended this functionality to support more granular environment-specific configurations. Environment variables are stored in the `./env` directory, where multiple `.env` files exist for different deployment environments (stages): - -- `.env` - Loaded for any environment -- `.env.production` - Loaded for the production stage -- `.env.staging` - Loaded for the staging stage - -### How Environment Variables Are Loaded -We use **dotenvx** to manage and load environment variables. This allows us to take advantage of its features, such as **variable interpolation** (i.e., using other environment variables within variable values). - -### Validation with Zod -Environment variables are validated using **Zod** schemas, ensuring that all required variables are present and have valid values. The validation logic can be found in the file `src/config/env-config.schema.ts`. - -We use two separate Zod schemas: -- **Static Build-Time Schema**: Validates variables at build time. If any variables are missing or invalid during the build process, the build will fail. -- **Dynamic Server Runtime Schema**: Validates variables at server startup. If any variables are missing or invalid at this stage, the server will fail to start. - -This validation ensures that both build and runtime configurations are secure and complete before the app runs. - -### App Configuration -App configurations, including environment variables, are located in the `src/config` directory. In our setup: -- **Environment configs** are handled separately from **hardcoded configs**. -- Hardcoded configs are organized by domain to maintain a clear structure and separation of concerns. - -### Sample Environment Variables -All environment variables required for the app, along with their expected structure and types, can be found in the `env/.env.sample` file. This sample file serves as a template for setting up your environment variables and ensures that all necessary variables are accounted for in each environment. - -By organizing environment variables and configuration this way, we ensure a consistent, safe, and scalable approach to managing different deployment environments. \ No newline at end of file diff --git a/apps/indexer/env/.env b/apps/indexer/env/.env deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/indexer/env/.env.mainnet b/apps/indexer/env/.env.mainnet new file mode 100644 index 000000000..ba4636212 --- /dev/null +++ b/apps/indexer/env/.env.mainnet @@ -0,0 +1 @@ +ACTIVE_CHAIN=akash \ No newline at end of file diff --git a/apps/indexer/env/.env.production b/apps/indexer/env/.env.production new file mode 100644 index 000000000..fa6b6e04d --- /dev/null +++ b/apps/indexer/env/.env.production @@ -0,0 +1,2 @@ +HEALTH_CHECKS_ENABLED=true +STANDBY=false \ No newline at end of file diff --git a/apps/indexer/env/.env.sample b/apps/indexer/env/.env.sample new file mode 100644 index 000000000..722f04005 --- /dev/null +++ b/apps/indexer/env/.env.sample @@ -0,0 +1,7 @@ +NETWORK=sandbox +ACTIVE_CHAIN= +AKASH_DATABASE_CS= +AKASH_SANDBOX_DATABASE_CS= +AKASH_TESTNET_DATABASE_CS= +KEEP_CACHE= +DATA_FOLDER= diff --git a/apps/indexer/env/.env.sandbox b/apps/indexer/env/.env.sandbox new file mode 100644 index 000000000..8fb0a0a5d --- /dev/null +++ b/apps/indexer/env/.env.sandbox @@ -0,0 +1 @@ +ACTIVE_CHAIN=akashSandbox diff --git a/apps/indexer/env/.env.testnet b/apps/indexer/env/.env.testnet new file mode 100644 index 000000000..6a5374f18 --- /dev/null +++ b/apps/indexer/env/.env.testnet @@ -0,0 +1 @@ +ACTIVE_CHAIN=akashTestnet diff --git a/apps/indexer/package.json b/apps/indexer/package.json index e17a06c8c..6b23c4c92 100644 --- a/apps/indexer/package.json +++ b/apps/indexer/package.json @@ -15,15 +15,16 @@ "main": "server.js", "scripts": { "build": "webpack --config webpack.prod.js", + "dev": "npm run start", "format": "prettier --write ./*.{js,json} **/*.{ts,js,json}", "lint": "eslint .", "start": "webpack --mode development --config webpack.dev.js --watch", - "dev": "npm run start", "test": "jest" }, "dependencies": { "@akashnetwork/akash-api": "^1.3.0", "@akashnetwork/database": "*", + "@akashnetwork/env-loader": "*", "@cosmjs/crypto": "^0.32.4", "@cosmjs/encoding": "^0.32.4", "@cosmjs/math": "^0.32.4", diff --git a/apps/indexer/src/index.ts b/apps/indexer/src/index.ts index f44c6266f..e6963fa69 100644 --- a/apps/indexer/src/index.ts +++ b/apps/indexer/src/index.ts @@ -1,3 +1,5 @@ +import "@akashnetwork/env-loader"; + import { activeChain, chainDefinitions } from "@akashnetwork/database/chainDefinitions"; import * as Sentry from "@sentry/node"; import express from "express"; diff --git a/doc/apps-configuration.md b/doc/apps-configuration.md new file mode 100644 index 000000000..c67e93e7a --- /dev/null +++ b/doc/apps-configuration.md @@ -0,0 +1,106 @@ +# Environment Variables & Configuration + +## Overview + +Both the **Next.js frontend** and **Node.js backend** applications use environment variables to manage configuration. This guide provides a unified approach to managing these variables, along with **Zod**-based validation for ensuring correctness across both apps. + +### Environment Files + +We use `.env*` files for managing environment variables, with a consistent loading order across both applications. These files help separate environment-specific configurations from sensitive information. + +### Next.js Environment Variables Behavior + +In addition to the logic described below environment variables in the **Next.js app** follow the standard Next.js behavior, as documented in the [Next.js environment variables documentation](https://nextjs.org/docs/basic-features/environment-variables). This means that files like `.env.local` or `.env.production` will be automatically loaded based on the environment in which the app is running, ensuring smooth transitions between development, staging, and production environments. + +### Loading Order + +The environment variables are loaded in the following order for both **Next.js** and **Node.js** applications: + +1. **System Environment Variables**: Variables set in the system environment (e.g., through CI/CD pipelines) take the highest precedence and are never overridden by values from `.env` files. +2. **`env/.env.local`**: Contains sensitive values specific to local development. This file is not tracked by Git and should contain secrets for local use. +3. **`env/.env`**: Contains default values applicable to all environments. This file is included in version control and should not contain sensitive data. +4. **`env/.env.${DEPLOYMENT_ENV}`**: Contains values specific to the deployment environment (e.g., production, staging). This file is loaded based on the `DEPLOYMENT_ENV` variable. +5. **`env/.env.${NETWORK}`**: Contains values specific to the network environment (e.g., mainnet, testnet). This file is loaded based on the `NETWORK` variable. + +### Variable Precedence + +- Variables loaded from higher-priority sources (like system environment variables or `.env.local`) will override those defined in lower-priority files (such as `.env` or `.env.production`). + +### Configuration Structure + +All application configurations should be organized and stored in specific files within the app directories: + +- **Configuration Files**: + - Configurations must be placed in `apps/*/config/.config.ts` files for each module-specific configuration. + - Environment variables should retain their original **SCREAMING_CASE** names and be parsed/validated directly using **Zod** schemas, without renaming. + +- **Split by Domain**: + - Configurations should be **split by application domain**. This helps maintain clarity and separation of concerns, with configurations logically grouped based on the features or domains they pertain to (e.g., database, authentication, API endpoints). + +### Environment Variables + +- **Separation of Environment Variables**: + - Environment variables must be separated from hardcoded configuration values to keep sensitive or environment-specific data outside of the codebase. + - Use `.env*` files as described in the **Loading Order** section, and ensure all environment variables are **validated** using schemas before they are used within the application. + +### Zod-Based Validation + +Both the **Next.js** and **Node.js** apps use **Zod** schemas to validate environment variables, ensuring that all required variables are present and have valid values. Validation is applied at two stages: + +- **Build-Time Validation** (Next.js only): In the **Next.js** app, variables are validated at build time using **Zod** schemas defined in the `src/config/env-config.schema.ts` file. If any required variables are missing or invalid, the build will fail. + +- **Runtime Validation** (All Apps): Both **Next.js** and **Node.js** applications perform **runtime validation** when the server starts. This ensures that all critical environment variables are present and valid before the server launches. If any required variables are missing or incorrect, the server will fail to start. + +### Example of Environment Variable Validation with Zod + +```typescript +// apps/config/env.config.ts +import { z } from "zod"; + +// Define the schema for environment variables +const envSchema = z.object({ + LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).optional().default("info"), + DATABASE_HOST: z.string(), + DATABASE_USER: z.string(), + SECRET_KEY: z.string(), +}); + +// Parse and validate the environment variables +export const envConfig = envSchema.parse(process.env); + +// Access the variables +console.log(envConfig.LOG_LEVEL); +console.log(envConfig.DATABASE_HOST); +``` + +### Sample Environment Variables + +Here’s an example `.env` file that corresponds to the validation schema above: + +```bash +# .env (shared across environments) +LOG_LEVEL=info +DATABASE_HOST=https://db.example.com +DATABASE_USER=myUser +SECRET_KEY=MY_SECRET_KEY + +# .env.local (development-specific, not included in version control) +DATABASE_HOST=http://localhost:5432 +DATABASE_USER=localUser +SECRET_KEY=LOCAL_SECRET_KEY + +# .env.production (production-specific) +DATABASE_HOST=https://prod-db.example.com +SECRET_KEY=PROD_SECRET_KEY +``` + +### Sample Environment Variables Template + +A template for setting up the required environment variables is provided in the `env/.env.sample` file for both types of applications. This file contains examples of all the necessary environment variables. + +By following this approach, we ensure a secure, scalable, and consistent configuration process for managing environment variables in both **Next.js** and **Node.js** applications, with robust validation through **Zod** and clear separation of configurations by application domain. + +### Disclaimer + +If you find any inconsistencies in the codebase compared to this documentation, please raise an issue or create a pull request to update the codebase accordingly. This documentation serves as the source of truth for managing environment variables and configurations across the applications. + diff --git a/package-lock.json b/package-lock.json index 97c2f7336..6f5620eb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "@akashnetwork/akash-api": "^1.3.0", "@akashnetwork/akashjs": "^0.10.0", "@akashnetwork/database": "*", + "@akashnetwork/env-loader": "*", "@akashnetwork/http-sdk": "*", "@casl/ability": "^6.7.1", "@chain-registry/assets": "^1.64.79", @@ -455,6 +456,7 @@ "dependencies": { "@akashnetwork/akash-api": "^1.3.0", "@akashnetwork/database": "*", + "@akashnetwork/env-loader": "*", "@cosmjs/crypto": "^0.32.4", "@cosmjs/encoding": "^0.32.4", "@cosmjs/math": "^0.32.4",