Feat: Add file tree, add include and exclude prefix for auto ingestion (

#639)

* Feat: Add knowledge file tree, add include and exclude list

Signed-off-by: Daishan Peng <[email protected]>

apiclient/types/knowledgesource.go

@@ -39,9 +39,11 @@ type KnowledgeSource struct {
 }
 
 type KnowledgeSourceManifest struct {
-	SyncSchedule         string `json:"syncSchedule,omitempty"`
-	AutoApprove          *bool  `json:"autoApprove,omitempty"`
-	KnowledgeSourceInput `json:",inline"`
+	SyncSchedule          string   `json:"syncSchedule,omitempty"`
+	AutoApprove           *bool    `json:"autoApprove,omitempty"`
+	FilePathPrefixInclude []string `json:"filePathPrefixInclude,omitempty"`
+	FilePathPrefixExclude []string `json:"filePathPrefixExclude,omitempty"`
+	KnowledgeSourceInput  `json:",inline"`
 }
 
 type KnowledgeSourceList List[KnowledgeSource]

pkg/controller/handlers/knowledgefile/knowledgefile.go

@@ -99,6 +99,24 @@ func (h *Handler) IngestFile(req router.Request, _ router.Response) error {
 		}
 	}
 
+	// Check approval
+	if file.Spec.Approved == nil {
+		switch {
+		case source.Spec.Manifest.AutoApprove != nil && *source.Spec.Manifest.AutoApprove:
+			file.Spec.Approved = typed.Pointer(true)
+		case isFileMatchPrefixPattern(file.Spec.FileName, source.Spec.Manifest.FilePathPrefixExclude):
+			file.Spec.Approved = typed.Pointer(false)
+		case isFileMatchPrefixPattern(file.Spec.FileName, source.Spec.Manifest.FilePathPrefixInclude):
+			file.Spec.Approved = typed.Pointer(true)
+		}
+
+		if file.Spec.Approved != nil {
+			if err := req.Client.Update(req.Ctx, file); err != nil {
+				return err
+			}
+		}
+	}
+
 	if file.Status.State.IsTerminal() && !shouldReIngest(file) {
 		return nil
 	}
@@ -111,16 +129,6 @@ func (h *Handler) IngestFile(req router.Request, _ router.Response) error {
 		}
 	}
 
-	// Check approval
-	if file.Spec.Approved == nil {
-		if source.Spec.Manifest.AutoApprove != nil && *source.Spec.Manifest.AutoApprove {
-			file.Spec.Approved = typed.Pointer(true)
-			if err := req.Client.Update(req.Ctx, file); err != nil {
-				return err
-			}
-		}
-	}
-
 	if file.Spec.Approved == nil || !*file.Spec.Approved {
 		// Not approved, wait for user action
 		return nil
@@ -395,3 +403,13 @@ func (h *Handler) Cleanup(req router.Request, _ router.Response) error {
 	}
 	return nil
 }
+
+func isFileMatchPrefixPattern(filePath string, patterns []string) bool {
+	for _, pattern := range patterns {
+		if strings.HasPrefix(filePath, pattern) {
+			return true
+		}
+	}
+
+	return false
+}