install by recipe

README.md

@@ -197,6 +197,22 @@ The dependency options are listed below:
 | `.[tools]`       | Install dependencies for dedicated tools, such as quality classifiers.                       |
 | `.[sandbox]`     | Install all dependencies for sandbox.                                                        |
 
+- Install dependencies for specific OPs
+
+With the growth of the number of OPs, the dependencies of all OPs becomes very heavy. Instead of using the command `pip install -v -e .[sci]` to install all dependencies,
+we provide two alternative, lighter options:
+
+  - Automatic Minimal Dependency Installation: During the execution of Data-Juicer, minimal dependencies will be automatically installed. This allows for immediate execution, but may potentially lead to dependency conflicts.
+
+  - Manual Minimal Dependency Installation: To manually install minimal dependencies tailored to a specific execution configuration, run the following command:
+    ```shell
+    # only for installation from source
+    python tools/dj_install.py --config path_to_your_data-juicer_config_file
+
+    # use command line tool
+    dj-install --config path_to_your_data-juicer_config_file
+    ```
+
 ### Using pip
 
 - Run the following command to install the latest released `data_juicer` using `pip`:

README_ZH.md

@@ -178,6 +178,21 @@ pip install -v -e .[tools] # 安装部分工具库的依赖
 | `.[tools]`       | 安装专用工具库（如质量分类器）所需的依赖项        |
 | `.[sandbox]`     | 安装沙盒实验室的基础依赖                 |
 
+* 只安装部分算子依赖
+
+随着OP数量的增长，所有OP的依赖变得很重。为此，我们提供了两个替代的、更轻量的选项，作为使用命令`pip install -v -e .[sci]`安装所有依赖的替代：
+
+  * 自动最小依赖安装：在执行Data-Juicer的过程中，将自动安装最小依赖。也就是说你可以直接执行，但这种方式可能会导致一些依赖冲突。
+
+  * 手动最小依赖安装：可以通过如下指令手动安装适合特定执行配置的最小依赖：
+    ```shell
+    # 适用于从源码安装
+    python tools/dj_install.py --config path_to_your_data-juicer_config_file
+
+    # 使用命令行工具
+    dj-install --config path_to_your_data-juicer_config_file
+    ```
+
 ### 使用 pip 安装
 
 * 运行以下命令用 `pip` 安装 `data_juicer` 的最新发布版本：

data_juicer/utils/auto_install_mapping.py

@@ -10,99 +10,74 @@
     'simhash': ['simhash-pybind'],
 }
 
-# Packages to corresponding ops that require them
-PKG_TO_OPS = {
-    'torch': [
-        'image_aesthetics_filter', 'image_nsfw_filter',
-        'image_text_matching_filter', 'image_text_similarity_filter',
-        'image_watermark_filter', 'phrase_grounding_recall_filter',
-        'video_aesthetics_filter', 'video_frames_text_similarity_filter',
-        'video_nsfw_filter', 'video_tagging_from_frames_filter',
-        'video_watermark_filter', 'generate_qa_from_text_mapper',
-        'generate_qa_from_examples_mapper', 'image_captioning_mapper',
-        'image_diffusion_mapper', 'image_tagging_mapper',
-        'optimize_query_mapper', 'optimize_response_mapper',
-        'optimize_qa_mapper', 'video_captioning_from_frames_mapper',
-        'video_captioning_from_summarizer_mapper',
-        'video_captioning_from_video_mapper',
-        'video_tagging_from_audio_mapper', 'video_tagging_from_frames_mapper'
+# Extra packages required by each op
+OPS_TO_PKG = {
+    'video_aesthetics_filter':
+    ['simple-aesthetics-predictor', 'torch', 'transformers'],
+    'document_simhash_deduplicator': ['simhash-pybind'],
+    'nlpcda_zh_mapper': ['nlpcda'],
+    'image_aesthetics_filter':
+    ['simple-aesthetics-predictor', 'torch', 'transformers'],
+    'video_nsfw_filter': ['torch', 'transformers'],
+    'video_face_blur_mapper': ['opencv-python'],
+    'stopwords_filter': ['sentencepiece'],
+    'fix_unicode_mapper': ['ftfy'],
+    'token_num_filter': ['transformers'],
+    'optimize_qa_mapper': ['torch', 'transformers', 'vllm'],
+    'video_motion_score_filter': ['opencv-python'],
+    'image_tagging_mapper': ['ram', 'torch'],
+    'video_resize_aspect_ratio_mapper': ['ffmpeg-python'],
+    'video_captioning_from_audio_mapper': [
+        'accelerate', 'einops', 'tiktoken', 'transformers',
+        'transformers_stream_generator'
     ],
-    'torchaudio': [
-        'video_captioning_from_summarizer_mapper',
-        'video_tagging_from_audio_mapper'
+    'clean_html_mapper': ['selectolax'],
+    'video_tagging_from_audio_mapper': ['torch', 'torchaudio', 'transformers'],
+    'image_deduplicator': ['imagededup'],
+    'image_diffusion_mapper':
+    ['diffusers', 'simhash-pybind', 'torch', 'transformers'],
+    'image_text_similarity_filter': ['torch', 'transformers'],
+    'alphanumeric_filter': ['transformers'],
+    'image_nsfw_filter': ['torch', 'transformers'],
+    'image_watermark_filter': ['torch', 'transformers'],
+    'ray_image_deduplicator': ['imagededup'],
+    'video_captioning_from_frames_mapper':
+    ['simhash-pybind', 'torch', 'transformers'],
+    'video_tagging_from_frames_filter': ['torch'],
+    'video_resize_resolution_mapper': ['ffmpeg-python'],
+    'optimize_query_mapper': ['torch', 'transformers', 'vllm'],
+    'sentence_split_mapper': ['nltk'],
+    'image_text_matching_filter': ['torch', 'transformers'],
+    'phrase_grounding_recall_filter': ['nltk', 'torch', 'transformers'],
+    'video_split_by_scene_mapper': ['scenedetect[opencv]'],
+    'image_face_blur_mapper': ['opencv-python'],
+    'image_face_ratio_filter': ['opencv-python'],
+    'document_minhash_deduplicator': ['scipy'],
+    'flagged_words_filter': ['sentencepiece'],
+    'language_id_score_filter': ['fasttext-wheel'],
+    'words_num_filter': ['sentencepiece'],
+    'chinese_convert_mapper': ['opencc'],
+    'video_frames_text_similarity_filter': ['torch', 'transformers'],
+    'generate_qa_from_text_mapper': ['torch', 'transformers', 'vllm'],
+    'video_ffmpeg_wrapped_mapper': ['ffmpeg-python'],
+    'image_captioning_mapper': ['simhash-pybind', 'torch', 'transformers'],
+    'video_ocr_area_ratio_filter': ['easyocr'],
+    'video_captioning_from_video_mapper':
+    ['simhash-pybind', 'torch', 'transformers'],
+    'video_remove_watermark_mapper': ['opencv-python'],
+    'text_action_filter': ['spacy-pkuseg'],
+    'nlpaug_en_mapper': ['nlpaug'],
+    'word_repetition_filter': ['sentencepiece'],
+    'video_watermark_filter': ['torch'],
+    'video_captioning_from_summarizer_mapper': [
+        'accelerate', 'einops', 'simhash-pybind', 'tiktoken', 'torch',
+        'torchaudio', 'transformers', 'transformers_stream_generator'
     ],
-    'easyocr': ['video_ocr_area_ratio_filter'],
-    'fasttext-wheel': ['language_id_score_filter'],
-    'kenlm': ['perplexity_filter'],
-    'sentencepiece': [
-        'flagged_words_filter', 'perplexity_filter', 'stopwords_filter',
-        'word_repetition_filter', 'words_num_filter'
-    ],
-    'scipy': ['document_minhash_deduplicator'],
-    'ftfy': ['fix_unicode_mapper'],
-    'simhash-pybind': [
-        'document_simhash_deduplicator', 'image_captioning_mapper',
-        'image_diffusion_mapper', 'video_captioning_from_frames_mapper',
-        'video_captioning_from_summarizer_mapper',
-        'video_captioning_from_video_mapper'
-    ],
-    'selectolax': ['clean_html_mapper'],
-    'nlpaug': ['nlpaug_en_mapper'],
-    'nlpcda': ['nlpcda'],
-    'nltk': ['phrase_grounding_recall_filter', 'sentence_split_mapper'],
-    'transformers': [
-        'alphanumeric_filter', 'image_aesthetics_filter', 'image_nsfw_filter',
-        'image_text_matching_filter', 'image_text_similarity_filter',
-        'image_watermark_filter', 'phrase_grounding_recall_filter',
-        'token_num_filter', 'video_aesthetics_filter',
-        'video_frames_text_similarity_filter', 'video_nsfw_filter',
-        'generate_qa_from_text_mapper', 'generate_qa_from_examples_mapper',
-        'image_captioning_mapper', 'image_diffusion_mapper',
-        'optimize_query_mapper', 'optimize_response_mapper',
-        'optimize_qa_mapper', 'video_captioning_from_audio_mapper',
-        'video_captioning_from_frames_mapper',
-        'video_captioning_from_summarizer_mapper',
-        'video_captioning_from_video_mapper', 'video_tagging_from_audio_mapper'
-    ],
-    'transformers_stream_generator': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'einops': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'accelerate': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'tiktoken': [
-        'video_captioning_from_audio_mapper',
-        'video_captioning_from_summarizer_mapper'
-    ],
-    'opencc': ['chinese_convert_mapper'],
-    'imagededup': ['image_deduplicator', 'ray_image_deduplicator'],
-    'spacy-pkuseg': ['text_action_filter', 'text_entity_dependency_filter'],
-    'diffusers': ['image_diffusion_mapper'],
-    'simple-aesthetics-predictor':
-    ['image_aesthetics_filter', 'video_aesthetics_filter'],
-    'scenedetect[opencv]': ['video_split_by_scene_mapper'],
-    'ffmpeg-python': [
-        'audio_ffmpeg_wrapped_mapper', 'video_ffmpeg_wrapped_mapper',
-        'video_resize_aspect_ratio_mapper', 'video_resize_resolution_mapper'
-    ],
-    'opencv-python': [
-        'image_face_ratio_filter', 'video_motion_score_filter',
-        'image_face_blur_mapper', 'video_face_blur_mapper',
-        'video_remove_watermark_mapper'
-    ],
-    'vllm': [
-        'generate_qa_from_text_mapper',
-        'generate_qa_from_examples_mapper',
-        'optimize_query_mapper',
-        'optimize_response_mapper',
-        'optimize_qa_mapper',
-    ],
-    'rouge': ['generate_qa_from_examples_mapper'],
-    'ram': ['image_tagging_mapper', 'video_tagging_from_frames_mapper']
+    'audio_ffmpeg_wrapped_mapper': ['ffmpeg-python'],
+    'perplexity_filter': ['kenlm', 'sentencepiece'],
+    'generate_qa_from_examples_mapper':
+    ['rouge', 'torch', 'transformers', 'vllm'],
+    'video_tagging_from_frames_mapper': ['ram', 'torch'],
+    'text_entity_dependency_filter': ['spacy-pkuseg'],
+    'optimize_response_mapper': ['torch', 'transformers', 'vllm']
 }

docs/DeveloperGuide.md

@@ -209,7 +209,9 @@ __all__ = [
 ]
 ```
 
-4. Now you can use this new OP with custom arguments in your own config files!
+4. When an operator has package dependencies listed in `environments/science_requires.txt`, you need to add the corresponding dependency packages to the `OPS_TO_PKG` dictionary in `data_juicer/utils/auto_install_mapping.py` to support dependency installation at the operator level.
+
+5. Now you can use this new OP with custom arguments in your own config files!
 
 ```yaml
 # other configs
@@ -222,7 +224,7 @@ process:
       max_len: 1000
 ```
 
-5. (Strongly Recommend) It's better to add corresponding tests for your own OPs. For `TextLengthFilter` above, you would like to add `test_text_length_filter.py` into `tests/ops/filter/` directory as below.
+6. (Strongly Recommend) It's better to add corresponding tests for your own OPs. For `TextLengthFilter` above, you would like to add `test_text_length_filter.py` into `tests/ops/filter/` directory as below.
 
 ```python
 import unittest
@@ -244,7 +246,7 @@ if __name__ == '__main__':
     unittest.main()
 ```
 
-6. (Strongly Recommend) In order to facilitate the use of other users, we also need to update this new OP information to
+7. (Strongly Recommend) In order to facilitate the use of other users, we also need to update this new OP information to
 the corresponding documents, including the following docs:
    1. `configs/config_all.yaml`: this complete config file contains a list of all OPs and their arguments, serving as an
    important document for users to refer to all available OPs. Therefore, after adding the new OP, we need to add it to the process

docs/DeveloperGuide_ZH.md

@@ -202,7 +202,9 @@ __all__ = [
 ]
 ```
 
-4. 全部完成！现在您可以在自己的配置文件中使用新添加的算子：
+4. 算子有`environments/science_requires.txt`中列举的包依赖时，需要在`data_juicer/utils/auto_install_mapping.py`里的`OPS_TO_PKG`中添加对应的依赖包，以支持算子粒度的依赖安装。
+
+5. 全部完成！现在您可以在自己的配置文件中使用新添加的算子：
 
 ```yaml
 # other configs
@@ -215,7 +217,7 @@ process:
       max_len: 1000
 ```
 
-5. （强烈推荐）最好为新添加的算子进行单元测试。对于上面的 `TextLengthFilter` 算子，建议在 `tests/ops/filter/` 中实现如 `test_text_length_filter.py` 的测试文件：
+6. （强烈推荐）最好为新添加的算子进行单元测试。对于上面的 `TextLengthFilter` 算子，建议在 `tests/ops/filter/` 中实现如 `test_text_length_filter.py` 的测试文件：
 
 ```python
 import unittest
@@ -238,7 +240,7 @@ if __name__ == '__main__':
     unittest.main()
 ```
 
-6. （强烈推荐）为了方便其他用户使用，我们还需要将新增的算子信息更新到相应的文档中，具体包括如下文档：
+7. （强烈推荐）为了方便其他用户使用，我们还需要将新增的算子信息更新到相应的文档中，具体包括如下文档：
    1. `configs/config_all.yaml`：该全集配置文件保存了所有算子及参数的一个列表，作为用户参考可用算子的一个重要文档。因此，在新增算子后，需要将其添加到该文档process列表里（按算子类型分组并按字母序排序）：
 
    ```yaml

setup.py

@@ -69,6 +69,7 @@ def get_install_requirements(require_f_paths, env_dir='environments'):
         'console_scripts': [
             'dj-process = data_juicer.tools.process_data:main',
             'dj-analyze = data_juicer.tools.analyze_data:main',
+            'dj-install = data_juicer.tools.dj_install:main',
         ]
     },
     install_requires=min_requires,

tools/dj_install.py

@@ -0,0 +1,65 @@
+import os
+import subprocess
+import sys
+import tempfile
+
+from loguru import logger
+
+from data_juicer.config import init_configs
+from data_juicer.utils.auto_install_mapping import OPS_TO_PKG
+
+require_version_paths = ['./environments/science_requires.txt']
+
+
+def main():
+    cfg = init_configs()
+
+    # get the ops in the recipe
+    op_names = [list(op.keys())[0] for op in cfg.process]
+    recipe_reqs = []
+    for op_name in op_names:
+        recipe_reqs.extend(OPS_TO_PKG[op_name])
+    recipe_reqs = list(set(recipe_reqs))
+
+    # get the package version limit of Data-Juicer
+    version_map, reqs = {}, []
+    for path in require_version_paths:
+        if not os.path.exists(path):
+            logger.warning(f'target file does not exist: {path}')
+        else:
+            with open(path, 'r', encoding='utf-8') as fin:
+                reqs += [x.strip() for x in fin.read().splitlines()]
+    for req in reqs:
+        clean_req = req.replace('<',
+                                ' ').replace('>',
+                                             ' ').replace('=',
+                                                          ' ').split(' ')[0]
+        version_map[clean_req] = req
+
+    # generate require file for the recipe
+    with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file:
+        temp_file_path = temp_file.name
+        for req in recipe_reqs:
+            if req in version_map:
+                temp_file.write(version_map[req] + '\n')
+            else:
+                temp_file.write(req + '\n')
+
+    # install by calling 'pip install -r ...'
+    try:
+        subprocess.check_call(
+            [sys.executable, '-m', 'pip', 'install', '-r', temp_file_path])
+        logger.info('Requirements were installed successfully.')
+    except subprocess.CalledProcessError as e:
+        logger.info(
+            f'An error occurred while installing the requirements: {e}')
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+        sys.exit(1)
+    finally:
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+
+
+if __name__ == '__main__':
+    main()