feat: add support for loading customformat from templates

docs/docs/configuration/config-file.md

@@ -32,17 +32,18 @@ Configarr supports two types of templates:
    - Can be overridden using `trashGuideUrl` in config.yml
    - See [Trash Radarr](https://github.com/TRaSH-Guides/Guides/tree/master/docs/json/radarr/quality-profiles) and [Trash Sonarr](https://github.com/TRaSH-Guides/Guides/tree/master/docs/json/sonarr/quality-profiles) for more information
 
-## Custom Formats
+## Custom Formats Definitions
 
 Custom formats can be defined in two ways:
 
 1. **Direct in config.yml**: Define custom formats directly in your configuration file
 2. **Separate files**: Store custom formats in separate files in the `localCustomFormatsPath` directory
+3. **Local templates**: Store custom formats in local templates folder which can be included per instance (at the moment only for local templates and not recyclarr git templates)
 
 Example custom format definition:
 
 ```yaml title="config.yml"
-# ...
+# Directly in the main config.yml
 customFormatDefinitions:
   - trash_id: custom-de-only # Unique identifier
     trash_scores:
@@ -60,6 +61,25 @@ customFormatDefinitions:
 # ...
 ```
 
+```yaml title="local-templates/template1.yml"
+# or in templates which can be included per instance
+customFormatDefinitions:
+  - trash_id: custom-de-only2 # Unique identifier
+    trash_scores:
+      default: -10000 # Default score for this format
+    trash_description: "Language: German Only 2"
+    name: "Language: Not German"
+    includeCustomFormatWhenRenaming: false
+    specifications:
+      - name: Not German Language
+        implementation: LanguageSpecification
+        negate: true
+        required: false
+        fields:
+          value: 4
+# ...
+```
+
 ## Configuration Files Reference
 
 ### config.yml

examples/full/templates/sonarr-cf.yml

@@ -15,3 +15,18 @@ custom_formats:
     quality_profiles:
       - name: ExampleProfile
         score: -10000
+
+customFormatDefinitions:
+  - trash_id: sonarr-cf
+    trash_scores:
+      default: -10000
+    trash_description: "Language: German Only 2"
+    name: "Language: Not German 2"
+    includeCustomFormatWhenRenaming: false
+    specifications:
+      - name: Not German Language
+        implementation: LanguageSpecification
+        negate: true
+        required: false
+        fields:
+          value: 4

src/custom-formats.test.ts

@@ -0,0 +1,166 @@
+import fs from "node:fs";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import * as config from "./config";
+import { calculateCFsToManage, loadCustomFormatDefinitions, loadLocalCfs, mergeCfSources } from "./custom-formats";
+import { loadTrashCFs } from "./trash-guide";
+import { CFProcessing } from "./types/common.types";
+import { ConfigCustomFormatList } from "./types/config.types";
+import { TrashCF } from "./types/trashguide.types";
+import * as util from "./util";
+
+describe("CustomFormats", () => {
+  let customCF: TrashCF;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mock("node:fs");
+
+    customCF = {
+      trash_id: "custom-size-more-40gb",
+      trash_scores: {
+        default: -10000,
+      },
+      trash_description: "Size: Block sizes over 40GB",
+      name: "Size: Block More 40GB",
+      includeCustomFormatWhenRenaming: false,
+      specifications: [
+        {
+          name: "Size",
+          implementation: "SizeSpecification",
+          negate: false,
+          required: true,
+          fields: {
+            min: 1,
+            max: 9,
+          },
+        },
+      ],
+    };
+  });
+
+  describe("loadLocalCfs", () => {
+    it("should return null when no local path is configured", async () => {
+      vi.spyOn(config, "getConfig").mockReturnValue({ localCustomFormatsPath: undefined });
+
+      const result = await loadLocalCfs();
+      expect(result).toBeNull();
+    });
+
+    it("should return null when configured path doesn't exist", async () => {
+      vi.spyOn(config, "getConfig").mockReturnValue({ localCustomFormatsPath: "/fake/path" });
+      vi.spyOn(fs, "existsSync").mockReturnValue(false);
+
+      const result = await loadLocalCfs();
+      expect(result).toBeNull();
+    });
+
+    it("should load and process JSON files from configured path", async () => {
+      vi.spyOn(config, "getConfig").mockReturnValue({ localCustomFormatsPath: "/valid/path" });
+      vi.spyOn(fs, "existsSync").mockReturnValue(true);
+
+      vi.spyOn(fs, "readdirSync").mockReturnValue(["test.json"] as any);
+      //vi.spyOn(fs, "readFileSync").mockImplementationOnce(() => "{}");
+
+      vi.spyOn(util, "loadJsonFile").mockReturnValueOnce(customCF);
+
+      const result = await loadLocalCfs();
+      expect(result).not.toBeNull();
+      expect(result!.carrIdMapping.size).toBe(1);
+      expect(result!.carrIdMapping.get(customCF.trash_id)).not.toBeNull();
+    });
+  });
+
+  describe("mergeCfSources", () => {
+    it("should merge multiple CF sources correctly", () => {
+      const source1: CFProcessing = {
+        carrIdMapping: new Map([["id1", { carrConfig: { configarr_id: "id1", name: "CF1" }, requestConfig: {} }]]),
+        cfNameToCarrConfig: new Map([["CF1", { configarr_id: "id1", name: "CF1" }]]),
+      };
+
+      const source2: CFProcessing = {
+        carrIdMapping: new Map([["id2", { carrConfig: { configarr_id: "id2", name: "CF2" }, requestConfig: {} }]]),
+        cfNameToCarrConfig: new Map([["CF2", { configarr_id: "id2", name: "CF2" }]]),
+      };
+
+      const result = mergeCfSources([source1, source2, null]);
+
+      expect(result.carrIdMapping.size).toBe(2);
+      expect(result.cfNameToCarrConfig.size).toBe(2);
+      expect(result.carrIdMapping.has("id1")).toBeTruthy();
+      expect(result.carrIdMapping.has("id2")).toBeTruthy();
+    });
+  });
+
+  describe("calculateCFsToManage", () => {
+    it("should collect all trash IDs from custom format list", () => {
+      const yaml: ConfigCustomFormatList = {
+        custom_formats: [
+          { trash_ids: ["t1", "t2"], assign_scores_to: [{ name: "default", score: 100 }] },
+          { trash_ids: ["t3"], assign_scores_to: [{ name: "default", score: 100 }] },
+          { trash_ids: ["t2"], assign_scores_to: [{ name: "default", score: 100 }] }, // Duplicate to test Set behavior
+        ],
+      };
+
+      const result = calculateCFsToManage(yaml);
+
+      expect(result.size).toBe(3);
+      expect(result.has("t1")).toBeTruthy();
+      expect(result.has("t2")).toBeTruthy();
+      expect(result.has("t3")).toBeTruthy();
+    });
+  });
+
+  describe("loadCustomFormatDefinitions", () => {
+    it("should load and merge (trash CFDs", async () => {
+      const mockTrashCFs: CFProcessing = {
+        carrIdMapping: new Map([["trash1", { carrConfig: { configarr_id: "trash1" }, requestConfig: {} }]]),
+        cfNameToCarrConfig: new Map(),
+      };
+
+      vi.mock("./trash-guide");
+      vi.mocked(loadTrashCFs).mockResolvedValue(mockTrashCFs);
+      vi.spyOn(config, "getConfig").mockReturnValue({ localCustomFormatsPath: undefined });
+
+      const result = await loadCustomFormatDefinitions("RADARR", []);
+
+      expect(result.carrIdMapping.size).toBe(1);
+      expect(result.carrIdMapping.has("trash1")).toBeTruthy();
+    });
+
+    it("should load and merge (additional CFDs)", async () => {
+      const mockTrashCFs: CFProcessing = {
+        carrIdMapping: new Map([["trash1", { carrConfig: { configarr_id: "trash1" }, requestConfig: {} }]]),
+        cfNameToCarrConfig: new Map(),
+      };
+
+      vi.mock("./trash-guide");
+      vi.mocked(loadTrashCFs).mockResolvedValue(mockTrashCFs);
+      vi.spyOn(config, "getConfig").mockReturnValue({ localCustomFormatsPath: undefined });
+
+      const result = await loadCustomFormatDefinitions("RADARR", [customCF]);
+
+      expect(result.carrIdMapping.size).toBe(2);
+      expect(result.carrIdMapping.has("trash1")).toBeTruthy();
+    });
+
+    it("should load and merge (config CFDs)", async () => {
+      const mockTrashCFs: CFProcessing = {
+        carrIdMapping: new Map(),
+        cfNameToCarrConfig: new Map(),
+      };
+
+      const clonedCFD: TrashCF = JSON.parse(JSON.stringify(customCF));
+      clonedCFD.trash_id = "trash2";
+      clonedCFD.name = "Trash2";
+
+      vi.mock("./trash-guide");
+      vi.mocked(loadTrashCFs).mockResolvedValue(mockTrashCFs);
+      vi.spyOn(config, "getConfig").mockReturnValue({ localCustomFormatsPath: undefined, customFormatDefinitions: [customCF] });
+
+      const result = await loadCustomFormatDefinitions("RADARR", [clonedCFD]);
+
+      expect(result.carrIdMapping.size).toBe(2);
+      expect(result.carrIdMapping.has("trash2")).toBeTruthy();
+    });
+  });
+});

src/custom-formats.ts

@@ -4,8 +4,9 @@ import { MergedCustomFormatResource } from "./__generated__/mergedTypes";
 import { getUnifiedClient } from "./clients/unified-client";
 import { getConfig } from "./config";
 import { logger } from "./logger";
-import { CFProcessing, ConfigarrCF } from "./types/common.types";
-import { ConfigCustomFormatList } from "./types/config.types";
+import { loadTrashCFs } from "./trash-guide";
+import { ArrType, CFProcessing, ConfigarrCF } from "./types/common.types";
+import { ConfigCustomFormatList, CustomFormatDefinitions } from "./types/config.types";
 import { TrashCF } from "./types/trashguide.types";
 import { IS_DRY_RUN, IS_LOCAL_SAMPLE_MODE, compareCustomFormats, loadJsonFile, mapImportCfToRequestCf, toCarrCF } from "./util";
 
@@ -155,18 +156,25 @@ export const loadLocalCfs = async (): Promise<CFProcessing | null> => {
 };
 
 export const loadCFFromConfig = (): CFProcessing | null => {
-  // TODO typings
   const defs = getConfig().customFormatDefinitions;
 
   if (defs == null) {
-    logger.debug(`No CustomFormat definitions defined.`);
+    logger.debug(`No local config CustomFormat definitions defined.`);
+    return null;
+  }
+
+  return mapCustomFormatDefinitions(defs);
+};
+
+export const mapCustomFormatDefinitions = (customFormatDefinitions: CustomFormatDefinitions): CFProcessing | null => {
+  if (customFormatDefinitions == null) {
     return null;
   }
 
   const carrIdToObject = new Map<string, { carrConfig: ConfigarrCF; requestConfig: MergedCustomFormatResource }>();
   const cfNameToCarrObject = new Map<string, ConfigarrCF>();
 
-  for (const def of defs) {
+  for (const def of customFormatDefinitions) {
     const cfD = toCarrCF(def);
 
     carrIdToObject.set(cfD.configarr_id, {
@@ -185,6 +193,18 @@ export const loadCFFromConfig = (): CFProcessing | null => {
   };
 };
 
+export const loadCustomFormatDefinitions = async (arrType: ArrType, additionalCFDs: CustomFormatDefinitions) => {
+  const trashCFs = await loadTrashCFs(arrType);
+  const localFileCFs = await loadLocalCfs();
+  const configCFDs = loadCFFromConfig();
+
+  logger.debug(
+    `Loaded ${trashCFs.carrIdMapping.size} TrashCFs, ${localFileCFs?.carrIdMapping.size} LocalCFs, ${configCFDs?.carrIdMapping.size} ConfigCFs, ${additionalCFDs.length} AdditionalCFs`,
+  );
+
+  return mergeCfSources([trashCFs, localFileCFs, configCFDs, mapCustomFormatDefinitions(additionalCFDs)]);
+};
+
 export const calculateCFsToManage = (yaml: ConfigCustomFormatList) => {
   const cfTrashToManage: Set<string> = new Set();
 

src/index.ts

@@ -4,7 +4,7 @@ import fs from "node:fs";
 import { MergedCustomFormatResource } from "./__generated__/mergedTypes";
 import { configureApi, getUnifiedClient, unsetApi } from "./clients/unified-client";
 import { getConfig, validateConfig } from "./config";
-import { calculateCFsToManage, loadCFFromConfig, loadLocalCfs, loadServerCustomFormats, manageCf, mergeCfSources } from "./custom-formats";
+import { calculateCFsToManage, loadCustomFormatDefinitions, loadServerCustomFormats, manageCf } from "./custom-formats";
 import { loadLocalRecyclarrTemplate } from "./local-importer";
 import { logHeading, logger } from "./logger";
 import { calculateQualityDefinitionDiff, loadQualityDefinitionFromServer } from "./quality-definitions";
@@ -14,12 +14,17 @@ import {
   cloneTrashRepo,
   loadQPFromTrash,
   loadQualityDefinitionFromTrash,
-  loadSonarrTrashCFs,
   transformTrashQPCFs,
   transformTrashQPToTemplate,
 } from "./trash-guide";
 import { ArrType, CFProcessing, MappedMergedTemplates } from "./types/common.types";
-import { ConfigQualityProfile, InputConfigArrInstance, InputConfigIncludeItem, MergedConfigInstance } from "./types/config.types";
+import {
+  ConfigQualityProfile,
+  CustomFormatDefinitions,
+  InputConfigArrInstance,
+  InputConfigIncludeItem,
+  MergedConfigInstance,
+} from "./types/config.types";
 import { TrashQualityDefintion } from "./types/trashguide.types";
 import { DEBUG_CREATE_FILES, IS_DRY_RUN } from "./util";
 
@@ -147,11 +152,20 @@ const mergeConfigsAndTemplates = async (
 
   recylarrMergedTemplates.quality_profiles = filterInvalidQualityProfiles(recylarrMergedTemplates.quality_profiles);
 
-  const trashCFs = await loadSonarrTrashCFs(arrType);
+  /* 
+  TODO: do we want to load all available local templates or only the included ones in the instance?
+  Example: we have a local template folder which we can always traverse. So we could load every CF defined there.
+  But then we could also have in theory conflicted CF IDs if user want to define same CF in different templates.
+  How to handle overwrite? Maybe also support overriding CFs defined in Trash or something?
+  */
+  const localTemplateCFDs = Array.from(localTemplateMap.values()).reduce((p, c) => {
+    if (c.customFormatDefinitions) {
+      p.push(...c.customFormatDefinitions);
+    }
+    return p;
+  }, [] as CustomFormatDefinitions);
 
-  const localFileCFs = await loadLocalCfs();
-  const configCFDefinition = loadCFFromConfig();
-  const mergedCFs = mergeCfSources([trashCFs, localFileCFs, configCFDefinition]);
+  const mergedCFs = await loadCustomFormatDefinitions(arrType, localTemplateCFDs);
 
   // merge profiles from recyclarr templates into one
   const qualityProfilesMerged = recylarrMergedTemplates.quality_profiles.reduce((p, c) => {

src/trash-guide.ts

@@ -22,7 +22,16 @@ export const cloneTrashRepo = async () => {
   logger.info(`TrashGuide repo: ref[${cloneResult.ref}], hash[${cloneResult.hash}], path[${cloneResult.localPath}]`);
 };
 
-export const loadSonarrTrashCFs = async (arrType: ArrType): Promise<CFProcessing> => {
+export const loadTrashCFs = async (arrType: ArrType): Promise<CFProcessing> => {
+  if (arrType !== "RADARR" && arrType !== "SONARR") {
+    logger.debug(`Unsupported arrType: ${arrType}. Skipping TrashCFs.`);
+
+    return {
+      carrIdMapping: new Map(),
+      cfNameToCarrConfig: new Map(),
+    };
+  }
+
   const trashRepoPath = "./repos/trash-guides";
   const trashJsonDir = "docs/json";
   const trashRadarrPath = `${trashJsonDir}/radarr`;
@@ -61,7 +70,7 @@ export const loadSonarrTrashCFs = async (arrType: ArrType): Promise<CFProcessing
     }
   }
 
-  logger.info(`Trash CFs: ${carrIdToObject.size}`);
+  logger.debug(`Trash CFs: ${carrIdToObject.size}`);
 
   return {
     carrIdMapping: carrIdToObject,

src/types/common.types.ts

@@ -44,10 +44,12 @@ export type ImportCF = OmitTyped<MergedCustomFormatResource, "specifications"> &
   specifications?: TCM[] | null;
 };
 
-export type ConfigarrCF = {
+export type ConfigarrCFMeta = {
   configarr_id: string;
   configarr_scores?: TrashCF["trash_scores"];
-} & ImportCF;
+};
+
+export type ConfigarrCF = ConfigarrCFMeta & ImportCF;
 
 export type CFProcessing = {
   carrIdMapping: Map<
@@ -61,7 +63,7 @@ export type CFProcessing = {
 };
 
 export type MappedTemplates = Partial<
-  Pick<InputConfigArrInstance, "quality_definition" | "custom_formats" | "include" | "quality_profiles">
+  Pick<InputConfigArrInstance, "quality_definition" | "custom_formats" | "include" | "quality_profiles" | "customFormatDefinitions">
 >;
 
 export type MappedMergedTemplates = MappedTemplates & Required<Pick<MappedTemplates, "custom_formats" | "quality_profiles">>;