feat: optimize envs and add support for custom root for data #117

.dockerignore

@@ -7,3 +7,5 @@ repos
 dockerrepos
 playwright-report
 test-results
+config.yml
+secrets.yml

.env.template

@@ -1,6 +1,8 @@
+#ROOT_PATH=/app
+#CUSTOM_REPO_ROOT=/app/repos
 #CONFIG_LOCATION=/app/config/config.yml
 #SECRETS_LOCATION=/app/config/secrets.yml
-DRY_RUN=true # not fully supported yet
-LOAD_LOCAL_SAMPLES=false
-DEBUG_CREATE_FILES=false
-LOG_LEVEL=info
+#DRY_RUN=true # not fully supported yet
+#LOAD_LOCAL_SAMPLES=false
+#DEBUG_CREATE_FILES=false
+#LOG_LEVEL=info

Dockerfile

@@ -29,8 +29,6 @@ COPY esbuild.ts ./
 RUN pnpm run build
 
 FROM base AS dev
-ENV CONFIG_LOCATION=/app/config/config.yml
-ENV SECRETS_LOCATION=/app/config/secrets.yml
 # manually mount src etc
 
 CMD [ "pnpm", "start" ]
@@ -46,8 +44,5 @@ RUN apk add --no-cache libstdc++ dumb-init git
 
 COPY --from=builder /app/bundle.cjs /app/index.js
 
-ENV CONFIG_LOCATION=/app/config/config.yml
-ENV SECRETS_LOCATION=/app/config/secrets.yml
-
 # Run with dumb-init to not start node with PID=1, since Node.js was not designed to run as PID 1
 CMD ["dumb-init", "node", "index.js"]

Dockerfile-deno.Dockerfile

@@ -16,8 +16,6 @@ COPY index.ts esbuild.ts ./
 RUN deno --allow-env --allow-read --allow-run esbuild.ts
 
 FROM base AS dev
-ENV CONFIG_LOCATION=/app/config/config.yml
-ENV SECRETS_LOCATION=/app/config/secrets.yml
 ENV DENO_DIR=/app/.deno_cache
 # manually mount src etc
 
@@ -34,8 +32,6 @@ RUN apk add --no-cache libstdc++ dumb-init git
 
 COPY --from=builder /app/bundle.cjs /app/index.cjs
 
-ENV CONFIG_LOCATION=/app/config/config.yml
-ENV SECRETS_LOCATION=/app/config/secrets.yml
 ENV DENO_DIR=/app/.deno_cache
 
 # Compile cache / modify for multi-user

docs/docs/configuration/environment-variables.md

@@ -0,0 +1,37 @@
+---
+sidebar_position: 2
+title: Environment Variables
+description: "Learn about the environment variables used in our application configuration."
+keywords: [environment variables, configuration, setup]
+---
+
+# Environment Variables
+
+This document outlines the available environment variables for configuring Configarr besides the config files.
+Each variable can be set to customize the behavior of the application.
+
+## Available Environment Variables
+
+| Variable Name        | Default Value             | Required | Description                                                                                 |
+| -------------------- | ------------------------- | -------- | ------------------------------------------------------------------------------------------- |
+| `LOG_LEVEL`          | `"info"`                  | No       | Sets the logging level. Options are `trace`, `debug`, `info`, `warn`, `error`, and `fatal`. |
+| `CONFIG_LOCATION`    | `"./config/config.yml"`   | No       | Specifies the path to the configuration file.                                               |
+| `SECRETS_LOCATION`   | `"./config/secrets.yml"`  | No       | Specifies the path to the secrets file.                                                     |
+| `CUSTOM_REPO_ROOT`   | `"./repos"`               | No       | Defines the root directory for custom repositories.                                         |
+| `ROOT_PATH`          | Current working directory | No       | Sets the root path for the application. Defaults to the current working directory.          |
+| `DRY_RUN`            | `"false"`                 | No       | When set to `"true"`, runs the application in dry run mode without making changes.          |
+| `LOAD_LOCAL_SAMPLES` | `"false"`                 | No       | If `"true"`, loads local sample data for testing purposes.                                  |
+| `DEBUG_CREATE_FILES` | `"false"`                 | No       | Enables debugging for file creation processes when set to `"true"`.                         |
+
+## Usage
+
+To use these environment variables, set them in your shell or include them in your deployment configuration via docker or kubernetes.
+
+## Examples
+
+- For example you change the default path for all configs, repos with the `ROOT_PATH` variables.
+  As default it would store them inside the application directory (in the container this is `/app`)
+
+## References
+
+Check the `.env.template` file in the repository [Github](https://github.com/raydak-labs/configarr/blob/main/.env.template)

docs/docs/configuration/experimental-support.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 2
+sidebar_position: 3
 title: Experimental Support
 description: "Experimental and testing support for other *Arr tools"
 keywords: [configarr configuration, yaml config, custom formats, expermintal, whisparr, readarr]

docs/docs/configuration/scheduled.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 3
+sidebar_position: 4
 title: Scheduling
 description: "How to run configarr regulary/schedueld"
 keywords: [configarr configuration, schedule, scheduler, regular, cron]

examples/full/.gitignore

@@ -1,2 +1,3 @@
 !config/*.yml
 dockerrepos/
+data/

package.json

@@ -23,7 +23,8 @@
     "pino-pretty": "13.0.0",
     "simple-git": "3.27.0",
     "tsx": "4.19.2",
-    "yaml": "2.6.1"
+    "yaml": "2.6.1",
+    "zod": "^3.24.1"
   },
   "devDependencies": {
     "@hyrious/esbuild-plugin-commonjs": "0.2.4",

src/config.ts

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync } from "node:fs";
 import yaml from "yaml";
+import { getHelpers } from "./env";
 import { logger } from "./logger";
 import {
   ConfigArrInstance,
@@ -12,10 +13,6 @@ import {
   InputConfigSchema,
   MergedConfigInstance,
 } from "./types/config.types";
-import { ROOT_PATH } from "./util";
-
-const CONFIG_LOCATION = process.env.CONFIG_LOCATION ?? `${ROOT_PATH}/config.yml`;
-const SECRETS_LOCATION = process.env.SECRETS_LOCATION ?? `${ROOT_PATH}/secrets.yml`;
 
 let config: ConfigSchema;
 let secrets: any;
@@ -50,12 +47,14 @@ export const getConfig = (): ConfigSchema => {
     return config;
   }
 
-  if (!existsSync(CONFIG_LOCATION)) {
-    logger.error(`Config file in location "${CONFIG_LOCATION}" does not exists.`);
+  const configLocation = getHelpers().configLocation;
+
+  if (!existsSync(configLocation)) {
+    logger.error(`Config file in location "${configLocation}" does not exists.`);
     throw new Error("Config file not found.");
   }
 
-  const file = readFileSync(CONFIG_LOCATION, "utf8");
+  const file = readFileSync(configLocation, "utf8");
 
   const inputConfig = yaml.parse(file, { customTags: [secretsTag, envTag] }) as InputConfigSchema;
 
@@ -65,12 +64,14 @@ export const getConfig = (): ConfigSchema => {
 };
 
 export const readConfigRaw = (): object => {
-  if (!existsSync(CONFIG_LOCATION)) {
-    logger.error(`Config file in location "${CONFIG_LOCATION}" does not exists.`);
+  const configLocation = getHelpers().configLocation;
+
+  if (!existsSync(configLocation)) {
+    logger.error(`Config file in location "${configLocation}" does not exists.`);
     throw new Error("Config file not found.");
   }
 
-  const file = readFileSync(CONFIG_LOCATION, "utf8");
+  const file = readFileSync(configLocation, "utf8");
 
   const inputConfig = yaml.parse(file, { customTags: [secretsTag, envTag] });
 
@@ -82,12 +83,14 @@ export const getSecrets = () => {
     return secrets;
   }
 
-  if (!existsSync(SECRETS_LOCATION)) {
-    logger.error(`Secret file in location "${SECRETS_LOCATION}" does not exists.`);
+  const secretLocation = getHelpers().secretLocation;
+
+  if (!existsSync(secretLocation)) {
+    logger.error(`Secret file in location "${secretLocation}" does not exists.`);
     throw new Error("Secret file not found.");
   }
 
-  const file = readFileSync(SECRETS_LOCATION, "utf8");
+  const file = readFileSync(secretLocation, "utf8");
   config = yaml.parse(file);
   return config;
 };

src/custom-formats.ts

@@ -3,12 +3,13 @@ import path from "node:path";
 import { MergedCustomFormatResource } from "./__generated__/mergedTypes";
 import { getUnifiedClient } from "./clients/unified-client";
 import { getConfig } from "./config";
+import { getEnvs } from "./env";
 import { logger } from "./logger";
 import { loadTrashCFs } from "./trash-guide";
 import { ArrType, CFProcessing, ConfigarrCF } from "./types/common.types";
 import { ConfigCustomFormatList, CustomFormatDefinitions } from "./types/config.types";
 import { TrashCF } from "./types/trashguide.types";
-import { IS_DRY_RUN, IS_LOCAL_SAMPLE_MODE, compareCustomFormats, loadJsonFile, mapImportCfToRequestCf, toCarrCF } from "./util";
+import { compareCustomFormats, loadJsonFile, mapImportCfToRequestCf, toCarrCF } from "./util";
 
 export const deleteAllCustomFormats = async () => {
   const api = getUnifiedClient();
@@ -21,7 +22,7 @@ export const deleteAllCustomFormats = async () => {
 };
 
 export const loadServerCustomFormats = async (): Promise<MergedCustomFormatResource[]> => {
-  if (IS_LOCAL_SAMPLE_MODE) {
+  if (getEnvs().LOAD_LOCAL_SAMPLES) {
     return loadJsonFile<MergedCustomFormatResource[]>(path.resolve(__dirname, "../tests/samples/cfs.json"));
   }
   const api = getUnifiedClient();
@@ -61,7 +62,7 @@ export const manageCf = async (
         logger.info(`Found mismatch for ${tr.requestConfig.name}: ${comparison.changes}`);
 
         try {
-          if (IS_DRY_RUN) {
+          if (getEnvs().DRY_RUN) {
             logger.info(`DryRun: Would update CF: ${existingCf.id} - ${existingCf.name}`);
             updatedCFs.push(existingCf);
           } else {
@@ -83,7 +84,7 @@ export const manageCf = async (
     } else {
       // Create
       try {
-        if (IS_DRY_RUN) {
+        if (getEnvs().DRY_RUN) {
           logger.info(`Would create CF: ${tr.requestConfig.name}`);
         } else {
           const createResult = await api.createCustomFormat(tr.requestConfig);
@@ -114,6 +115,7 @@ export const manageCf = async (
 
   return { createCFs, updatedCFs, validCFs, errorCFs };
 };
+
 export const loadLocalCfs = async (): Promise<CFProcessing | null> => {
   const config = getConfig();
 

src/env.ts

@@ -0,0 +1,72 @@
+import path from "node:path";
+import { z } from "zod";
+
+const DEFAULT_ROOT_PATH = path.resolve(process.cwd());
+
+const schema = z.object({
+  // NODE_ENV: z.enum(["production", "development", "test"] as const),
+  LOG_LEVEL: z
+    .enum(["trace", "debug", "info", "warn", "error", "fatal"] as const)
+    .optional()
+    .default("info"),
+  CONFIG_LOCATION: z.string().optional(),
+  SECRETS_LOCATION: z.string().optional(),
+  // TODO: deprecate?
+  CUSTOM_REPO_ROOT: z.string().optional(),
+  ROOT_PATH: z.string().optional().default(DEFAULT_ROOT_PATH),
+  DRY_RUN: z
+    .string()
+    .toLowerCase()
+    .transform((x) => x === "true")
+    .pipe(z.boolean())
+    .default("false"),
+  LOAD_LOCAL_SAMPLES: z
+    .string()
+    .toLowerCase()
+    .transform((x) => x === "true")
+    .pipe(z.boolean())
+    .default("false"),
+  DEBUG_CREATE_FILES: z
+    .string()
+    .toLowerCase()
+    .transform((x) => x === "true")
+    .pipe(z.boolean())
+    .default("false"),
+});
+
+// declare global {
+//   // eslint-disable-next-line @typescript-eslint/no-namespace
+//   namespace NodeJS {
+//     // eslint-disable-next-line @typescript-eslint/no-empty-object-type
+//     interface ProcessEnv extends z.infer<typeof schema> {}
+//   }
+// }
+
+let envs: z.infer<typeof schema>;
+
+export function initEnvs() {
+  const parsed = schema.safeParse(process.env);
+
+  if (parsed.success === false) {
+    console.error("Invalid environment variables:", parsed.error.flatten().fieldErrors);
+    throw new Error("Invalid environment variables.");
+  }
+
+  envs = parsed.data;
+}
+
+export const getEnvs = () => {
+  if (envs) return envs;
+
+  envs = schema.parse(process.env);
+
+  return envs;
+};
+
+export const getHelpers = () => ({
+  configLocation: getEnvs().CONFIG_LOCATION ?? `${getEnvs().ROOT_PATH}/config/config.yml`,
+  secretLocation: getEnvs().SECRETS_LOCATION ?? `${getEnvs().ROOT_PATH}/config/secrets.yml`,
+  // TODO: check for different env name
+  repoPath: getEnvs().CUSTOM_REPO_ROOT ?? `${getEnvs().ROOT_PATH}/repos`,
+  // TODO: add stuff like isDryRun,...?
+});

src/index.ts

@@ -1,4 +1,7 @@
+// those must be run first!
 import "dotenv/config";
+import { getEnvs, initEnvs } from "./env";
+initEnvs();
 
 import fs from "node:fs";
 import { MergedCustomFormatResource } from "./__generated__/mergedTypes";
@@ -26,7 +29,6 @@ import {
   MergedConfigInstance,
 } from "./types/config.types";
 import { TrashQualityDefintion } from "./types/trashguide.types";
-import { DEBUG_CREATE_FILES, IS_DRY_RUN } from "./util";
 
 /**
  * Load data from trash, recyclarr, custom configs and merge.
@@ -250,7 +252,7 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => {
     const { changeMap, create, restData } = calculateQualityDefinitionDiff(serverQD, qdTrash, config.quality_definition?.preferred_ratio);
 
     if (changeMap.size > 0) {
-      if (IS_DRY_RUN) {
+      if (getEnvs().DRY_RUN) {
         logger.info("DryRun: Would update QualityDefinitions.");
       } else {
         logger.info(`Diffs in quality definitions found`, changeMap.values());
@@ -272,15 +274,15 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => {
   const serverQP = await loadQualityProfilesFromServer();
   const { changedQPs, create, noChanges } = await calculateQualityProfilesDiff(mergedCFs, config, serverQP, serverQD, serverCFs);
 
-  if (DEBUG_CREATE_FILES) {
+  if (getEnvs().DEBUG_CREATE_FILES) {
     create.concat(changedQPs).forEach((e, i) => {
       fs.writeFileSync(`debug/test${i}.json`, JSON.stringify(e, null, 2), "utf-8");
     });
   }
 
   logger.info(`QualityProfiles: Create: ${create.length}, Update: ${changedQPs.length}, Unchanged: ${noChanges.length}`);
 
-  if (!IS_DRY_RUN) {
+  if (!getEnvs().DRY_RUN) {
     for (const element of create) {
       try {
         const newProfile = await api.createQualityProfile(element);
@@ -306,7 +308,7 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => {
 };
 
 const run = async () => {
-  if (IS_DRY_RUN) {
+  if (getEnvs().DRY_RUN) {
     logger.info("DryRun: Running in dry-run mode!");
   }