add flag for specifying chunk size

mitodl · Jan 21, 2025 · 5e07494 · 5e07494
1 parent 5a3e772
commit 5e07494
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/vector_search/management/commands/generate_embeddings.py b/vector_search/management/commands/generate_embeddings.py
@@ -43,6 +43,18 @@ def add_arguments(self, parser):
             help="Skip embedding content files",
         )
 
+        parser.add_argument(
+            "--contentfile-chunk-size",
+            dest="contentfile_chunk_size",
+            action="store_true",
+            help=(
+                "Specify the chunk size for content file embeddings. "
+                "NOTE: if this is specified - the collection name for contentfiles "
+                "will be suffixed with the chunk size. "
+                "example - learn.content_files_512"
+            ),
+        )
+
         for object_type in sorted(LEARNING_RESOURCE_TYPES):
             parser.add_argument(
                 f"--{object_type}s",