Skip to content

Commit

Permalink
fix: change DirectoryComponent to filter file paths by types (#2391)
Browse files Browse the repository at this point in the history
* Refactor DirectoryComponent to filter file paths by types

* fix: sets types is_list to True

* chore: Remove Optional from load_directory method signature

* chore: Update return type annotation for load_directory method

* style(GroqModel.py): improve code readability by fixing indentation and adding a comment for type hint ignore in ChatGroq instantiation.

---------

Co-authored-by: Gabriel Luiz Freitas Almeida <[email protected]>
  • Loading branch information
carlosrcoelho and ogabrielluiz authored Jul 5, 2024
1 parent e536272 commit bc6b846
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions src/backend/base/langflow/components/data/Directory.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import List, Optional
from typing import List

from langflow.base.data.utils import parallel_load_data, parse_text_file_to_data, retrieve_file_paths
from langflow.custom import Component
Expand All @@ -23,6 +23,7 @@ class DirectoryComponent(Component):
name="types",
display_name="Types",
info="File types to load. Leave empty to load all types.",
is_list=True,
),
IntInput(
name="depth",
Expand Down Expand Up @@ -67,8 +68,9 @@ class DirectoryComponent(Component):
Output(display_name="Data", name="data", method="load_directory"),
]

def load_directory(self) -> List[Optional[Data]]:
def load_directory(self) -> List[Data]:
path = self.path
types = self.types or [] # self.types is already a list due to is_list=True
depth = self.depth
max_concurrency = self.max_concurrency
load_hidden = self.load_hidden
Expand All @@ -78,6 +80,10 @@ def load_directory(self) -> List[Optional[Data]]:

resolved_path = self.resolve_path(path)
file_paths = retrieve_file_paths(resolved_path, load_hidden, recursive, depth)

if types:
file_paths = [fp for fp in file_paths if any(fp.endswith(ext) for ext in types)]

loaded_data = []

if use_multithreading:
Expand All @@ -86,4 +92,4 @@ def load_directory(self) -> List[Optional[Data]]:
loaded_data = [parse_text_file_to_data(file_path, silent_errors) for file_path in file_paths]
loaded_data = list(filter(None, loaded_data))
self.status = loaded_data
return loaded_data
return loaded_data # type: ignore

0 comments on commit bc6b846

Please sign in to comment.