install by recipe

README.md

@@ -197,6 +197,18 @@ The dependency options are listed below:
 | `.[tools]`       | Install dependencies for dedicated tools, such as quality classifiers.                       |
 | `.[sandbox]`     | Install all dependencies for sandbox.                                                        |
 
+- Install dependencies for specific OPs
+
+With the growth of the number of OPs, the dependencies of all OPs becomes very heavy. Instead of using the command `pip install -v -e .[sci]` to install all dependencies,
+we provide two alternative, lighter options:
+
+  - Automatic Minimal Dependency Installation: During the execution of Data-Juicer, minimal dependencies will be automatically installed. This allows for immediate execution, but may potentially lead to dependency conflicts.
+
+  - Manual Minimal Dependency Installation: To manually install minimal dependencies tailored to a specific execution configuration, run the following command:
+    ```shell
+    python tools/install_by_recipe.py --config path_to_your_data-juicer_config_file
+    ```
+
 ### Using pip
 
 - Run the following command to install the latest released `data_juicer` using `pip`:

README_ZH.md

@@ -178,6 +178,17 @@ pip install -v -e .[tools] # 安装部分工具库的依赖
 | `.[tools]`       | 安装专用工具库（如质量分类器）所需的依赖项        |
 | `.[sandbox]`     | 安装沙盒实验室的基础依赖                 |
 
+* 只安装部分算子依赖
+
+随着OP数量的增长，所有OP的依赖变得很重。为此，我们提供了两个替代的、更轻量的选项，作为使用命令`pip install -v -e .[sci]`安装所有依赖的替代：
+
+  * 自动最小依赖安装：在执行Data-Juicer的过程中，将自动安装最小依赖。也就是说你可以直接执行，但这种方式可能会导致一些依赖冲突。
+
+  * 手动最小依赖安装：可以通过如下指令手动安装适合特定执行配置的最小依赖：
+    ```shell
+    python tools/install_by_recipe.py --config path_to_your_data-juicer_config_file
+    ```
+
 ### 使用 pip 安装
 
 * 运行以下命令用 `pip` 安装 `data_juicer` 的最新发布版本：

data_juicer/utils/auto_install_mapping.py

@@ -10,99 +10,74 @@
     'simhash': ['simhash-pybind'],
 }
 
-# Packages to corresponding ops that require them
-PKG_TO_OPS = {
-    'torch': [
-        'image_aesthetics_filter', 'image_nsfw_filter',
-        'image_text_matching_filter', 'image_text_similarity_filter',
-        'image_watermark_filter', 'phrase_grounding_recall_filter',
-        'video_aesthetics_filter', 'video_frames_text_similarity_filter',
-        'video_nsfw_filter', 'video_tagging_from_frames_filter',
-        'video_watermark_filter', 'generate_qa_from_text_mapper',
-        'generate_qa_from_examples_mapper', 'image_captioning_mapper',
-        'image_diffusion_mapper', 'image_tagging_mapper',
-        'optimize_query_mapper', 'optimize_response_mapper',
-        'optimize_qa_mapper', 'video_captioning_from_frames_mapper',
-        'video_captioning_from_summarizer_mapper',
-        'video_captioning_from_video_mapper',
-        'video_tagging_from_audio_mapper', 'video_tagging_from_frames_mapper'
-    ],
-    'torchaudio': [
-        'video_captioning_from_summarizer_mapper',
-        'video_tagging_from_audio_mapper'
-    ],
-    'easyocr': ['video_ocr_area_ratio_filter'],
-    'fasttext-wheel': ['language_id_score_filter'],
-    'kenlm': ['perplexity_filter'],
-    'sentencepiece': [
-        'flagged_words_filter', 'perplexity_filter', 'stopwords_filter',
-        'word_repetition_filter', 'words_num_filter'
-    ],
-    'scipy': ['document_minhash_deduplicator'],
-    'ftfy': ['fix_unicode_mapper'],
-    'simhash-pybind': [
-        'document_simhash_deduplicator', 'image_captioning_mapper',
-        'image_diffusion_mapper', 'video_captioning_from_frames_mapper',
-        'video_captioning_from_summarizer_mapper',
-        'video_captioning_from_video_mapper'
-    ],
-    'selectolax': ['clean_html_mapper'],
-    'nlpaug': ['nlpaug_en_mapper'],
+# Extra packages required by each op
+OPS_TO_PKG = {
+    'video_aesthetics_filter':
+    ['simple-aesthetics-predictor', 'torch', 'transformers'],
+    'document_simhash_deduplicator': ['simhash-pybind'],
     'nlpcda': ['nlpcda'],
-    'nltk': ['phrase_grounding_recall_filter', 'sentence_split_mapper'],
-    'transformers': [
-        'alphanumeric_filter', 'image_aesthetics_filter', 'image_nsfw_filter',
-        'image_text_matching_filter', 'image_text_similarity_filter',
-        'image_watermark_filter', 'phrase_grounding_recall_filter',
-        'token_num_filter', 'video_aesthetics_filter',
-        'video_frames_text_similarity_filter', 'video_nsfw_filter',
-        'generate_qa_from_text_mapper', 'generate_qa_from_examples_mapper',
-        'image_captioning_mapper', 'image_diffusion_mapper',
-        'optimize_query_mapper', 'optimize_response_mapper',
-        'optimize_qa_mapper', 'video_captioning_from_audio_mapper',
-        'video_captioning_from_frames_mapper',
-        'video_captioning_from_summarizer_mapper',
-        'video_captioning_from_video_mapper', 'video_tagging_from_audio_mapper'
-    ],
-    'transformers_stream_generator': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'einops': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'accelerate': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'tiktoken': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'opencc': ['chinese_convert_mapper'],
-    'imagededup': ['image_deduplicator', 'ray_image_deduplicator'],
-    'spacy-pkuseg': ['text_action_filter', 'text_entity_dependency_filter'],
-    'diffusers': ['image_diffusion_mapper'],
-    'simple-aesthetics-predictor':
-    ['image_aesthetics_filter', 'video_aesthetics_filter'],
-    'scenedetect[opencv]': ['video_split_by_scene_mapper'],
-    'ffmpeg-python': [
-        'audio_ffmpeg_wrapped_mapper', 'video_ffmpeg_wrapped_mapper',
-        'video_resize_aspect_ratio_mapper', 'video_resize_resolution_mapper'
-    ],
-    'opencv-python': [
-        'image_face_ratio_filter', 'video_motion_score_filter',
-        'image_face_blur_mapper', 'video_face_blur_mapper',
-        'video_remove_watermark_mapper'
+    'image_aesthetics_filter':
+    ['simple-aesthetics-predictor', 'torch', 'transformers'],
+    'video_nsfw_filter': ['torch', 'transformers'],
+    'video_face_blur_mapper': ['opencv-python'],
+    'stopwords_filter': ['sentencepiece'],
+    'fix_unicode_mapper': ['ftfy'],
+    'token_num_filter': ['transformers'],
+    'optimize_qa_mapper': ['torch', 'transformers', 'vllm'],
+    'video_motion_score_filter': ['opencv-python'],
+    'image_tagging_mapper': ['ram', 'torch'],
+    'video_resize_aspect_ratio_mapper': ['ffmpeg-python'],
+    'video_captioning_from_audio_mapper': [
+        'accelerate', 'einops', 'tiktoken', 'transformers',
+        'transformers_stream_generator'
     ],
-    'vllm': [
-        'generate_qa_from_text_mapper',
-        'generate_qa_from_examples_mapper',
-        'optimize_query_mapper',
-        'optimize_response_mapper',
-        'optimize_qa_mapper',
+    'clean_html_mapper': ['selectolax'],
+    'video_tagging_from_audio_mapper': ['torch', 'torchaudio', 'transformers'],
+    'image_deduplicator': ['imagededup'],
+    'image_diffusion_mapper':
+    ['diffusers', 'simhash-pybind', 'torch', 'transformers'],
+    'image_text_similarity_filter': ['torch', 'transformers'],
+    'alphanumeric_filter': ['transformers'],
+    'image_nsfw_filter': ['torch', 'transformers'],
+    'image_watermark_filter': ['torch', 'transformers'],
+    'ray_image_deduplicator': ['imagededup'],
+    'video_captioning_from_frames_mapper':
+    ['simhash-pybind', 'torch', 'transformers'],
+    'video_tagging_from_frames_filter': ['torch'],
+    'video_resize_resolution_mapper': ['ffmpeg-python'],
+    'optimize_query_mapper': ['torch', 'transformers', 'vllm'],
+    'sentence_split_mapper': ['nltk'],
+    'image_text_matching_filter': ['torch', 'transformers'],
+    'phrase_grounding_recall_filter': ['nltk', 'torch', 'transformers'],
+    'video_split_by_scene_mapper': ['scenedetect[opencv]'],
+    'image_face_blur_mapper': ['opencv-python'],
+    'image_face_ratio_filter': ['opencv-python'],
+    'document_minhash_deduplicator': ['scipy'],
+    'flagged_words_filter': ['sentencepiece'],
+    'language_id_score_filter': ['fasttext-wheel'],
+    'words_num_filter': ['sentencepiece'],
+    'chinese_convert_mapper': ['opencc'],
+    'video_frames_text_similarity_filter': ['torch', 'transformers'],
+    'generate_qa_from_text_mapper': ['torch', 'transformers', 'vllm'],
+    'video_ffmpeg_wrapped_mapper': ['ffmpeg-python'],
+    'image_captioning_mapper': ['simhash-pybind', 'torch', 'transformers'],
+    'video_ocr_area_ratio_filter': ['easyocr'],
+    'video_captioning_from_video_mapper':
+    ['simhash-pybind', 'torch', 'transformers'],
+    'video_remove_watermark_mapper': ['opencv-python'],
+    'text_action_filter': ['spacy-pkuseg'],
+    'nlpaug_en_mapper': ['nlpaug'],
+    'word_repetition_filter': ['sentencepiece'],
+    'video_watermark_filter': ['torch'],
+    'video_captioning_from_summarizer_mapper': [
+        'accelerate', 'einops', 'simhash-pybind', 'tiktoken', 'torch',
+        'torchaudio', 'transformers', 'transformers_stream_generator'
     ],
-    'rouge': ['generate_qa_from_examples_mapper'],
-    'ram': ['image_tagging_mapper', 'video_tagging_from_frames_mapper']
+    'audio_ffmpeg_wrapped_mapper': ['ffmpeg-python'],
+    'perplexity_filter': ['kenlm', 'sentencepiece'],
+    'generate_qa_from_examples_mapper':
+    ['rouge', 'torch', 'transformers', 'vllm'],
+    'video_tagging_from_frames_mapper': ['ram', 'torch'],
+    'text_entity_dependency_filter': ['spacy-pkuseg'],
+    'optimize_response_mapper': ['torch', 'transformers', 'vllm']
 }

tools/install_by_recipe.py

@@ -0,0 +1,65 @@
+import os
+import subprocess
+import sys
+import tempfile
+
+from loguru import logger
+
+from data_juicer.config import init_configs
+from data_juicer.utils.auto_install_mapping import OPS_TO_PKG
+
+require_version_paths = ['./environments/science_requires.txt']
+
+
+def main():
+    cfg = init_configs()
+
+    # get the ops in the recipe
+    op_names = [list(op.keys())[0] for op in cfg.process]
+    recipe_reqs = []
+    for op_name in op_names:
+        recipe_reqs.extend(OPS_TO_PKG[op_name])
+    recipe_reqs = list(set(recipe_reqs))
+
+    # get the package version limit of Data-Juicer
+    version_map, reqs = {}, []
+    for path in require_version_paths:
+        if not os.path.exists(path):
+            logger.warning(f'target file does not exist: {path}')
+        else:
+            with open(path, 'r', encoding='utf-8') as fin:
+                reqs += [x.strip() for x in fin.read().splitlines()]
+    for req in reqs:
+        clean_req = req.replace('<',
+                                ' ').replace('>',
+                                             ' ').replace('=',
+                                                          ' ').split(' ')[0]
+        version_map[clean_req] = req
+
+    # generate require file for the recipe
+    with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file:
+        temp_file_path = temp_file.name
+        for req in recipe_reqs:
+            if req in version_map:
+                temp_file.write(version_map[req] + '\n')
+            else:
+                temp_file.write(req + '\n')
+
+    # install by calling 'pip install -r ...'
+    try:
+        subprocess.check_call(
+            [sys.executable, '-m', 'pip', 'install', '-r', temp_file_path])
+        logger.info('Requirements were installed successfully.')
+    except subprocess.CalledProcessError as e:
+        logger.info(
+            f'An error occurred while installing the requirements: {e}')
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+        sys.exit(1)
+    finally:
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+
+
+if __name__ == '__main__':
+    main()