diff --git a/.dockerignore b/.dockerignore index b50ec40..346bf27 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,5 @@ repos dockerrepos playwright-report test-results +config.yml +secrets.yml diff --git a/.env.template b/.env.template index 3aa057e..9e58530 100644 --- a/.env.template +++ b/.env.template @@ -1,6 +1,8 @@ +#ROOT_PATH=/app +#CUSTOM_REPO_ROOT=/app/repos #CONFIG_LOCATION=/app/config/config.yml #SECRETS_LOCATION=/app/config/secrets.yml -DRY_RUN=true # not fully supported yet -LOAD_LOCAL_SAMPLES=false -DEBUG_CREATE_FILES=false -LOG_LEVEL=info +#DRY_RUN=true # not fully supported yet +#LOAD_LOCAL_SAMPLES=false +#DEBUG_CREATE_FILES=false +#LOG_LEVEL=info diff --git a/Dockerfile b/Dockerfile index 10f20ed..ee9dfa6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,6 @@ COPY esbuild.ts ./ RUN pnpm run build FROM base AS dev -ENV CONFIG_LOCATION=/app/config/config.yml -ENV SECRETS_LOCATION=/app/config/secrets.yml # manually mount src etc CMD [ "pnpm", "start" ] @@ -46,8 +44,5 @@ RUN apk add --no-cache libstdc++ dumb-init git COPY --from=builder /app/bundle.cjs /app/index.js -ENV CONFIG_LOCATION=/app/config/config.yml -ENV SECRETS_LOCATION=/app/config/secrets.yml - # Run with dumb-init to not start node with PID=1, since Node.js was not designed to run as PID 1 CMD ["dumb-init", "node", "index.js"] diff --git a/Dockerfile-deno.Dockerfile b/Dockerfile-deno.Dockerfile index 2822951..5a0b869 100644 --- a/Dockerfile-deno.Dockerfile +++ b/Dockerfile-deno.Dockerfile @@ -16,8 +16,6 @@ COPY index.ts esbuild.ts ./ RUN deno --allow-env --allow-read --allow-run esbuild.ts FROM base AS dev -ENV CONFIG_LOCATION=/app/config/config.yml -ENV SECRETS_LOCATION=/app/config/secrets.yml ENV DENO_DIR=/app/.deno_cache # manually mount src etc @@ -34,8 +32,6 @@ RUN apk add --no-cache libstdc++ dumb-init git COPY --from=builder /app/bundle.cjs /app/index.cjs -ENV CONFIG_LOCATION=/app/config/config.yml -ENV SECRETS_LOCATION=/app/config/secrets.yml ENV DENO_DIR=/app/.deno_cache # Compile cache / modify for multi-user diff --git a/docs/docs/configuration/environment-variables.md b/docs/docs/configuration/environment-variables.md new file mode 100644 index 0000000..9d72c2a --- /dev/null +++ b/docs/docs/configuration/environment-variables.md @@ -0,0 +1,37 @@ +--- +sidebar_position: 2 +title: Environment Variables +description: "Learn about the environment variables used in our application configuration." +keywords: [environment variables, configuration, setup] +--- + +# Environment Variables + +This document outlines the available environment variables for configuring Configarr besides the config files. +Each variable can be set to customize the behavior of the application. + +## Available Environment Variables + +| Variable Name | Default Value | Required | Description | +| -------------------- | ------------------------- | -------- | ------------------------------------------------------------------------------------------- | +| `LOG_LEVEL` | `"info"` | No | Sets the logging level. Options are `trace`, `debug`, `info`, `warn`, `error`, and `fatal`. | +| `CONFIG_LOCATION` | `"./config/config.yml"` | No | Specifies the path to the configuration file. | +| `SECRETS_LOCATION` | `"./config/secrets.yml"` | No | Specifies the path to the secrets file. | +| `CUSTOM_REPO_ROOT` | `"./repos"` | No | Defines the root directory for custom repositories. | +| `ROOT_PATH` | Current working directory | No | Sets the root path for the application. Defaults to the current working directory. | +| `DRY_RUN` | `"false"` | No | When set to `"true"`, runs the application in dry run mode without making changes. | +| `LOAD_LOCAL_SAMPLES` | `"false"` | No | If `"true"`, loads local sample data for testing purposes. | +| `DEBUG_CREATE_FILES` | `"false"` | No | Enables debugging for file creation processes when set to `"true"`. | + +## Usage + +To use these environment variables, set them in your shell or include them in your deployment configuration via docker or kubernetes. + +## Examples + +- For example you change the default path for all configs, repos with the `ROOT_PATH` variables. + As default it would store them inside the application directory (in the container this is `/app`) + +## References + +Check the `.env.template` file in the repository [Github](https://github.com/raydak-labs/configarr/blob/main/.env.template) diff --git a/docs/docs/configuration/experimental-support.md b/docs/docs/configuration/experimental-support.md index 21cab67..7f60861 100644 --- a/docs/docs/configuration/experimental-support.md +++ b/docs/docs/configuration/experimental-support.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 3 title: Experimental Support description: "Experimental and testing support for other *Arr tools" keywords: [configarr configuration, yaml config, custom formats, expermintal, whisparr, readarr] diff --git a/docs/docs/configuration/scheduled.md b/docs/docs/configuration/scheduled.md index f968b56..8e167b1 100644 --- a/docs/docs/configuration/scheduled.md +++ b/docs/docs/configuration/scheduled.md @@ -1,5 +1,5 @@ --- -sidebar_position: 3 +sidebar_position: 4 title: Scheduling description: "How to run configarr regulary/schedueld" keywords: [configarr configuration, schedule, scheduler, regular, cron] diff --git a/examples/full/.gitignore b/examples/full/.gitignore index dc67370..be39477 100644 --- a/examples/full/.gitignore +++ b/examples/full/.gitignore @@ -1,2 +1,3 @@ !config/*.yml dockerrepos/ +data/ diff --git a/package.json b/package.json index 858aa0d..98ff686 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "pino-pretty": "13.0.0", "simple-git": "3.27.0", "tsx": "4.19.2", - "yaml": "2.6.1" + "yaml": "2.6.1", + "zod": "^3.24.1" }, "devDependencies": { "@hyrious/esbuild-plugin-commonjs": "0.2.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50cd6d6..fbb5103 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: yaml: specifier: 2.6.1 version: 2.6.1 + zod: + specifier: ^3.24.1 + version: 3.24.1 devDependencies: '@hyrious/esbuild-plugin-commonjs': specifier: 0.2.4 @@ -2479,6 +2482,9 @@ packages: resolution: {integrity: sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==} engines: {node: '>=18'} + zod@3.24.1: + resolution: {integrity: sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==} + snapshots: '@ampproject/remapping@2.3.0': @@ -4786,3 +4792,5 @@ snapshots: yargs-parser: 21.1.1 yoctocolors-cjs@2.1.2: {} + + zod@3.24.1: {} diff --git a/src/config.ts b/src/config.ts index c577781..d2e4728 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,6 @@ import { existsSync, readFileSync } from "node:fs"; import yaml from "yaml"; +import { getHelpers } from "./env"; import { logger } from "./logger"; import { ConfigArrInstance, @@ -12,10 +13,6 @@ import { InputConfigSchema, MergedConfigInstance, } from "./types/config.types"; -import { ROOT_PATH } from "./util"; - -const CONFIG_LOCATION = process.env.CONFIG_LOCATION ?? `${ROOT_PATH}/config.yml`; -const SECRETS_LOCATION = process.env.SECRETS_LOCATION ?? `${ROOT_PATH}/secrets.yml`; let config: ConfigSchema; let secrets: any; @@ -50,12 +47,14 @@ export const getConfig = (): ConfigSchema => { return config; } - if (!existsSync(CONFIG_LOCATION)) { - logger.error(`Config file in location "${CONFIG_LOCATION}" does not exists.`); + const configLocation = getHelpers().configLocation; + + if (!existsSync(configLocation)) { + logger.error(`Config file in location "${configLocation}" does not exists.`); throw new Error("Config file not found."); } - const file = readFileSync(CONFIG_LOCATION, "utf8"); + const file = readFileSync(configLocation, "utf8"); const inputConfig = yaml.parse(file, { customTags: [secretsTag, envTag] }) as InputConfigSchema; @@ -65,12 +64,14 @@ export const getConfig = (): ConfigSchema => { }; export const readConfigRaw = (): object => { - if (!existsSync(CONFIG_LOCATION)) { - logger.error(`Config file in location "${CONFIG_LOCATION}" does not exists.`); + const configLocation = getHelpers().configLocation; + + if (!existsSync(configLocation)) { + logger.error(`Config file in location "${configLocation}" does not exists.`); throw new Error("Config file not found."); } - const file = readFileSync(CONFIG_LOCATION, "utf8"); + const file = readFileSync(configLocation, "utf8"); const inputConfig = yaml.parse(file, { customTags: [secretsTag, envTag] }); @@ -82,12 +83,14 @@ export const getSecrets = () => { return secrets; } - if (!existsSync(SECRETS_LOCATION)) { - logger.error(`Secret file in location "${SECRETS_LOCATION}" does not exists.`); + const secretLocation = getHelpers().secretLocation; + + if (!existsSync(secretLocation)) { + logger.error(`Secret file in location "${secretLocation}" does not exists.`); throw new Error("Secret file not found."); } - const file = readFileSync(SECRETS_LOCATION, "utf8"); + const file = readFileSync(secretLocation, "utf8"); config = yaml.parse(file); return config; }; diff --git a/src/custom-formats.ts b/src/custom-formats.ts index 96b781e..2b8c132 100644 --- a/src/custom-formats.ts +++ b/src/custom-formats.ts @@ -3,12 +3,13 @@ import path from "node:path"; import { MergedCustomFormatResource } from "./__generated__/mergedTypes"; import { getUnifiedClient } from "./clients/unified-client"; import { getConfig } from "./config"; +import { getEnvs } from "./env"; import { logger } from "./logger"; import { loadTrashCFs } from "./trash-guide"; import { ArrType, CFProcessing, ConfigarrCF } from "./types/common.types"; import { ConfigCustomFormatList, CustomFormatDefinitions } from "./types/config.types"; import { TrashCF } from "./types/trashguide.types"; -import { IS_DRY_RUN, IS_LOCAL_SAMPLE_MODE, compareCustomFormats, loadJsonFile, mapImportCfToRequestCf, toCarrCF } from "./util"; +import { compareCustomFormats, loadJsonFile, mapImportCfToRequestCf, toCarrCF } from "./util"; export const deleteAllCustomFormats = async () => { const api = getUnifiedClient(); @@ -21,7 +22,7 @@ export const deleteAllCustomFormats = async () => { }; export const loadServerCustomFormats = async (): Promise => { - if (IS_LOCAL_SAMPLE_MODE) { + if (getEnvs().LOAD_LOCAL_SAMPLES) { return loadJsonFile(path.resolve(__dirname, "../tests/samples/cfs.json")); } const api = getUnifiedClient(); @@ -61,7 +62,7 @@ export const manageCf = async ( logger.info(`Found mismatch for ${tr.requestConfig.name}: ${comparison.changes}`); try { - if (IS_DRY_RUN) { + if (getEnvs().DRY_RUN) { logger.info(`DryRun: Would update CF: ${existingCf.id} - ${existingCf.name}`); updatedCFs.push(existingCf); } else { @@ -83,7 +84,7 @@ export const manageCf = async ( } else { // Create try { - if (IS_DRY_RUN) { + if (getEnvs().DRY_RUN) { logger.info(`Would create CF: ${tr.requestConfig.name}`); } else { const createResult = await api.createCustomFormat(tr.requestConfig); @@ -114,6 +115,7 @@ export const manageCf = async ( return { createCFs, updatedCFs, validCFs, errorCFs }; }; + export const loadLocalCfs = async (): Promise => { const config = getConfig(); diff --git a/src/env.ts b/src/env.ts new file mode 100644 index 0000000..f8d744c --- /dev/null +++ b/src/env.ts @@ -0,0 +1,72 @@ +import path from "node:path"; +import { z } from "zod"; + +const DEFAULT_ROOT_PATH = path.resolve(process.cwd()); + +const schema = z.object({ + // NODE_ENV: z.enum(["production", "development", "test"] as const), + LOG_LEVEL: z + .enum(["trace", "debug", "info", "warn", "error", "fatal"] as const) + .optional() + .default("info"), + CONFIG_LOCATION: z.string().optional(), + SECRETS_LOCATION: z.string().optional(), + // TODO: deprecate? + CUSTOM_REPO_ROOT: z.string().optional(), + ROOT_PATH: z.string().optional().default(DEFAULT_ROOT_PATH), + DRY_RUN: z + .string() + .toLowerCase() + .transform((x) => x === "true") + .pipe(z.boolean()) + .default("false"), + LOAD_LOCAL_SAMPLES: z + .string() + .toLowerCase() + .transform((x) => x === "true") + .pipe(z.boolean()) + .default("false"), + DEBUG_CREATE_FILES: z + .string() + .toLowerCase() + .transform((x) => x === "true") + .pipe(z.boolean()) + .default("false"), +}); + +// declare global { +// // eslint-disable-next-line @typescript-eslint/no-namespace +// namespace NodeJS { +// // eslint-disable-next-line @typescript-eslint/no-empty-object-type +// interface ProcessEnv extends z.infer {} +// } +// } + +let envs: z.infer; + +export function initEnvs() { + const parsed = schema.safeParse(process.env); + + if (parsed.success === false) { + console.error("Invalid environment variables:", parsed.error.flatten().fieldErrors); + throw new Error("Invalid environment variables."); + } + + envs = parsed.data; +} + +export const getEnvs = () => { + if (envs) return envs; + + envs = schema.parse(process.env); + + return envs; +}; + +export const getHelpers = () => ({ + configLocation: getEnvs().CONFIG_LOCATION ?? `${getEnvs().ROOT_PATH}/config/config.yml`, + secretLocation: getEnvs().SECRETS_LOCATION ?? `${getEnvs().ROOT_PATH}/config/secrets.yml`, + // TODO: check for different env name + repoPath: getEnvs().CUSTOM_REPO_ROOT ?? `${getEnvs().ROOT_PATH}/repos`, + // TODO: add stuff like isDryRun,...? +}); diff --git a/src/index.ts b/src/index.ts index 8fd6d14..a043bdf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,7 @@ +// those must be run first! import "dotenv/config"; +import { getEnvs, initEnvs } from "./env"; +initEnvs(); import fs from "node:fs"; import { MergedCustomFormatResource } from "./__generated__/mergedTypes"; @@ -26,7 +29,6 @@ import { MergedConfigInstance, } from "./types/config.types"; import { TrashQualityDefintion } from "./types/trashguide.types"; -import { DEBUG_CREATE_FILES, IS_DRY_RUN } from "./util"; /** * Load data from trash, recyclarr, custom configs and merge. @@ -250,7 +252,7 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => { const { changeMap, create, restData } = calculateQualityDefinitionDiff(serverQD, qdTrash, config.quality_definition?.preferred_ratio); if (changeMap.size > 0) { - if (IS_DRY_RUN) { + if (getEnvs().DRY_RUN) { logger.info("DryRun: Would update QualityDefinitions."); } else { logger.info(`Diffs in quality definitions found`, changeMap.values()); @@ -272,7 +274,7 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => { const serverQP = await loadQualityProfilesFromServer(); const { changedQPs, create, noChanges } = await calculateQualityProfilesDiff(mergedCFs, config, serverQP, serverQD, serverCFs); - if (DEBUG_CREATE_FILES) { + if (getEnvs().DEBUG_CREATE_FILES) { create.concat(changedQPs).forEach((e, i) => { fs.writeFileSync(`debug/test${i}.json`, JSON.stringify(e, null, 2), "utf-8"); }); @@ -280,7 +282,7 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => { logger.info(`QualityProfiles: Create: ${create.length}, Update: ${changedQPs.length}, Unchanged: ${noChanges.length}`); - if (!IS_DRY_RUN) { + if (!getEnvs().DRY_RUN) { for (const element of create) { try { const newProfile = await api.createQualityProfile(element); @@ -306,7 +308,7 @@ const pipeline = async (value: InputConfigArrInstance, arrType: ArrType) => { }; const run = async () => { - if (IS_DRY_RUN) { + if (getEnvs().DRY_RUN) { logger.info("DryRun: Running in dry-run mode!"); } diff --git a/src/logger.ts b/src/logger.ts index 5e91c5f..3650e49 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,7 +1,6 @@ import { levels, pino } from "pino"; import pretty from "pino-pretty"; - -export const LOG_LEVEL = process.env.LOG_LEVEL ?? `info`; +import { getEnvs, getHelpers } from "./env"; const maxArrayLength = Object.values(levels.labels).reduce((maxLength, currentArray) => { return Math.max(maxLength, currentArray.length); @@ -31,7 +30,7 @@ const stream = pretty({ export const logger = pino( { - level: LOG_LEVEL, + level: getEnvs().LOG_LEVEL, // transport: { // target: "pino-pretty", // options: { @@ -53,3 +52,6 @@ export const logHeading = (title: string) => { logSeparator(); logger.info(""); }; + +// For debugging how envs have been loaded +logger.debug({ envs: getEnvs(), helpers: getHelpers() }, `Loaded following configuration from ENVs and mapped`); diff --git a/src/quality-definitions.ts b/src/quality-definitions.ts index 35ece83..ea2a19e 100644 --- a/src/quality-definitions.ts +++ b/src/quality-definitions.ts @@ -1,12 +1,13 @@ import path from "node:path"; import { MergedQualityDefinitionResource } from "./__generated__/mergedTypes"; import { getUnifiedClient } from "./clients/unified-client"; +import { getEnvs } from "./env"; import { logger } from "./logger"; import { TrashQualityDefintion, TrashQualityDefintionQuality } from "./types/trashguide.types"; -import { IS_LOCAL_SAMPLE_MODE, loadJsonFile } from "./util"; +import { loadJsonFile } from "./util"; export const loadQualityDefinitionFromServer = async (): Promise => { - if (IS_LOCAL_SAMPLE_MODE) { + if (getEnvs().LOAD_LOCAL_SAMPLES) { return loadJsonFile(path.resolve(__dirname, "../tests/samples/qualityDefinition.json")); } return await getUnifiedClient().getQualityDefinitions(); diff --git a/src/quality-profiles.ts b/src/quality-profiles.ts index 4d39dfa..65ae3a4 100644 --- a/src/quality-profiles.ts +++ b/src/quality-profiles.ts @@ -7,10 +7,11 @@ import { MergedQualityProfileResource, } from "./__generated__/mergedTypes"; import { getUnifiedClient } from "./clients/unified-client"; +import { getEnvs } from "./env"; import { logger } from "./logger"; import { CFProcessing } from "./types/common.types"; import { ConfigQualityProfile, ConfigQualityProfileItem, MergedConfigInstance } from "./types/config.types"; -import { IS_LOCAL_SAMPLE_MODE, cloneWithJSON, loadJsonFile, notEmpty, zip } from "./util"; +import { cloneWithJSON, loadJsonFile, notEmpty, zip } from "./util"; // merge CFs of templates and custom CFs into one mapping of QualityProfile -> CFs + Score export const mapQualityProfiles = ({ carrIdMapping }: CFProcessing, { custom_formats, quality_profiles }: MergedConfigInstance) => { @@ -69,7 +70,7 @@ export const mapQualityProfiles = ({ carrIdMapping }: CFProcessing, { custom_for }; export const loadQualityProfilesFromServer = async (): Promise => { - if (IS_LOCAL_SAMPLE_MODE) { + if (getEnvs().LOAD_LOCAL_SAMPLES) { return loadJsonFile(path.resolve(__dirname, `../tests/samples/quality_profiles.json`)); } const api = getUnifiedClient(); diff --git a/src/util.ts b/src/util.ts index 34e2fc3..6e05b8a 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,17 +2,12 @@ import { existsSync, mkdirSync, readFileSync } from "node:fs"; import path from "node:path"; import simpleGit, { CheckRepoActions } from "simple-git"; import { MergedCustomFormatResource } from "./__generated__/mergedTypes"; +import { getHelpers } from "./env"; import { logger } from "./logger"; import { ConfigarrCF, ImportCF, UserFriendlyField } from "./types/common.types"; import { TrashCF } from "./types/trashguide.types"; -export const IS_DRY_RUN = process.env.DRY_RUN === "true"; -export const IS_LOCAL_SAMPLE_MODE = process.env.LOAD_LOCAL_SAMPLES === "true"; -export const DEBUG_CREATE_FILES = process.env.DEBUG_CREATE_FILES === "true"; - -export const repoPath = path.resolve(process.env.CUSTOM_REPO_ROOT || "./repos"); - -const recyclarrConfigPath = `${repoPath}/recyclarr-config`; +const recyclarrConfigPath = `${getHelpers().repoPath}/recyclarr-config`; const recyclarrSonarrRoot = `${recyclarrConfigPath}/sonarr`; const recyclarrSonarrCFs = `${recyclarrSonarrRoot}/includes/custom-formats`; const recyclarrSonarrQDs = `${recyclarrSonarrRoot}/includes/quality-definitions`; @@ -23,8 +18,8 @@ const recyclarrRadarrCFs = `${recyclarrRadarrRoot}/includes/custom-formats`; const recyclarrRadarrQDs = `${recyclarrRadarrRoot}/includes/quality-definitions`; const recyclarrRadarrQPs = `${recyclarrRadarrRoot}/includes/quality-profiles`; +const trashRepoRoot = `${getHelpers().repoPath}/trash-guides`; const trashRepoPath = "docs/json"; -const trashRepoRoot = `${repoPath}/trash-guides`; const trashRepoSonarrRoot = `${trashRepoRoot}/${trashRepoPath}/sonarr`; const trashRepoRadarrRoot = `${trashRepoRoot}/${trashRepoPath}/radarr`; @@ -254,6 +249,7 @@ export const cloneGitRepo = async (localPath: string, gitUrl: string, revision: const r = await gitClient.checkIsRepo(CheckRepoActions.IS_REPO_ROOT); if (!r) { + logger.info(`Freshly cloned repository: '${gitUrl}' at '${revision}'`); await simpleGit().clone(gitUrl, rootPath); } @@ -266,6 +262,7 @@ export const cloneGitRepo = async (localPath: string, gitUrl: string, revision: const res = await gitClient.pull(); if (res.files.length > 0) { updated = true; + logger.info(`Repository updated to new commit: '${gitUrl}' at '${revision}'`); } }