deploy: 4b8b436

modelscope · Dec 9, 2024 · a365d21 · a365d21
1 parent d9142eb
commit a365d21
Show file tree

Hide file tree

Showing 170 changed files with 4,518 additions and 2,290 deletions.
diff --git a/.buildinfo b/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
-# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 4c41756fe8e61f33740e7839901685c2
+# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: c0644f6dcb399e85439a8e09f1d45c95
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/_modules/data_juicer.html b/_modules/data_juicer.html
@@ -1,18 +1,18 @@
 
 
 <!DOCTYPE html>
-<html class="writer-html5" lang="en">
+<html class="writer-html5" lang="en" data-content_root="../">
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>data_juicer &mdash; data_juicer 1.0.0 documentation</title>
+  <title>data_juicer &mdash; data_juicer 1.0.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
       <link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
 
 
-      <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js?v=af2ce170"></script>
-      <script src="../_static/doctools.js?v=888ff710"></script>
-      <script src="../_static/sphinx_highlight.js?v=4825356b"></script>
+      <script src="../_static/documentation_options.js?v=292eb321"></script>
+      <script src="../_static/doctools.js?v=9bcbadda"></script>
+      <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="../_static/js/theme.js"></script>
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" /> 
@@ -76,7 +76,7 @@
            <div itemprop="articleBody">
 
   <h1>Source code for data_juicer</h1><div class="highlight"><pre>
-<span></span><span class="n">__version__</span> <span class="o">=</span> <span class="s1">&#39;1.0.0&#39;</span>
+<span></span><span class="n">__version__</span> <span class="o">=</span> <span class="s1">&#39;1.0.1&#39;</span>
 
 <span class="kn">import</span> <span class="nn">os</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
@@ -123,12 +123,18 @@ <h1>Source code for data_juicer</h1><div class="highlight"><pre>
 <span class="n">_CUDA_DEVICE_COUNT</span> <span class="o">=</span> <span class="n">_cuda_device_count</span><span class="p">()</span>
 
 
-<div class="viewcode-block" id="cuda_device_count"><a class="viewcode-back" href="../data_juicer.html#data_juicer.cuda_device_count">[docs]</a><span class="k">def</span> <span class="nf">cuda_device_count</span><span class="p">():</span>
+<div class="viewcode-block" id="cuda_device_count">
+<a class="viewcode-back" href="../data_juicer.html#data_juicer.cuda_device_count">[docs]</a>
+<span class="k">def</span> <span class="nf">cuda_device_count</span><span class="p">():</span>
     <span class="k">return</span> <span class="n">_CUDA_DEVICE_COUNT</span></div>
 
 
-<div class="viewcode-block" id="is_cuda_available"><a class="viewcode-back" href="../data_juicer.html#data_juicer.is_cuda_available">[docs]</a><span class="k">def</span> <span class="nf">is_cuda_available</span><span class="p">():</span>
+
+<div class="viewcode-block" id="is_cuda_available">
+<a class="viewcode-back" href="../data_juicer.html#data_juicer.is_cuda_available">[docs]</a>
+<span class="k">def</span> <span class="nf">is_cuda_available</span><span class="p">():</span>
     <span class="k">return</span> <span class="n">_CUDA_DEVICE_COUNT</span> <span class="o">&gt;</span> <span class="mi">0</span></div>
+
 </pre></div>
 
            </div>

diff --git a/_modules/data_juicer/analysis/column_wise_analysis.html b/_modules/data_juicer/analysis/column_wise_analysis.html
@@ -1,18 +1,18 @@
 
 
 <!DOCTYPE html>
-<html class="writer-html5" lang="en">
+<html class="writer-html5" lang="en" data-content_root="../../../">
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>data_juicer.analysis.column_wise_analysis &mdash; data_juicer 1.0.0 documentation</title>
+  <title>data_juicer.analysis.column_wise_analysis &mdash; data_juicer 1.0.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=80d5e7a1" />
       <link rel="stylesheet" type="text/css" href="../../../_static/css/theme.css?v=e59714d7" />
 
 
-      <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js?v=af2ce170"></script>
-      <script src="../../../_static/doctools.js?v=888ff710"></script>
-      <script src="../../../_static/sphinx_highlight.js?v=4825356b"></script>
+      <script src="../../../_static/documentation_options.js?v=292eb321"></script>
+      <script src="../../../_static/doctools.js?v=9bcbadda"></script>
+      <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="../../../_static/js/theme.js"></script>
     <link rel="index" title="Index" href="../../../genindex.html" />
     <link rel="search" title="Search" href="../../../search.html" /> 
@@ -130,10 +130,14 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
     <span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">now_row</span><span class="p">),</span> <span class="nb">int</span><span class="p">(</span><span class="n">now_col</span><span class="p">),</span> <span class="n">grids</span>
 
 
-<div class="viewcode-block" id="ColumnWiseAnalysis"><a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis">[docs]</a><span class="k">class</span> <span class="nc">ColumnWiseAnalysis</span><span class="p">:</span>
+<div class="viewcode-block" id="ColumnWiseAnalysis">
+<a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis">[docs]</a>
+<span class="k">class</span> <span class="nc">ColumnWiseAnalysis</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Apply analysis on each column of stats respectively.&quot;&quot;&quot;</span>
 
-<div class="viewcode-block" id="ColumnWiseAnalysis.__init__"><a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.__init__">[docs]</a>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+<div class="viewcode-block" id="ColumnWiseAnalysis.__init__">
+<a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.__init__">[docs]</a>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                  <span class="n">dataset</span><span class="p">,</span>
                  <span class="n">output_path</span><span class="p">,</span>
                  <span class="n">overall_result</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -160,7 +164,10 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
 
         <span class="bp">self</span><span class="o">.</span><span class="n">save_stats_in_one_file</span> <span class="o">=</span> <span class="n">save_stats_in_one_file</span></div>
 
-<div class="viewcode-block" id="ColumnWiseAnalysis.analyze"><a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.analyze">[docs]</a>    <span class="k">def</span> <span class="nf">analyze</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">show_percentiles</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">show</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">skip_export</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+
+<div class="viewcode-block" id="ColumnWiseAnalysis.analyze">
+<a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.analyze">[docs]</a>
+    <span class="k">def</span> <span class="nf">analyze</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">show_percentiles</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">show</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">skip_export</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Apply analysis and draw the analysis figure for stats.</span>
 
@@ -268,7 +275,10 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
                 <span class="c1"># TODO: (fixme) the saved png sometime are blank</span>
                 <span class="n">plt</span><span class="o">.</span><span class="n">clf</span><span class="p">()</span></div>
 
-<div class="viewcode-block" id="ColumnWiseAnalysis.draw_hist"><a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.draw_hist">[docs]</a>    <span class="k">def</span> <span class="nf">draw_hist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ax</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">save_path</span><span class="p">,</span> <span class="n">percentiles</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">show</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+
+<div class="viewcode-block" id="ColumnWiseAnalysis.draw_hist">
+<a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.draw_hist">[docs]</a>
+    <span class="k">def</span> <span class="nf">draw_hist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ax</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">save_path</span><span class="p">,</span> <span class="n">percentiles</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">show</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Draw the histogram for the data.</span>
 
@@ -329,7 +339,10 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
             <span class="c1"># add a little rotation on labels of x axis to avoid overlapping</span>
             <span class="n">ax</span><span class="o">.</span><span class="n">tick_params</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="s1">&#39;x&#39;</span><span class="p">,</span> <span class="n">rotation</span><span class="o">=</span><span class="mi">25</span><span class="p">)</span></div>
 
-<div class="viewcode-block" id="ColumnWiseAnalysis.draw_box"><a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.draw_box">[docs]</a>    <span class="k">def</span> <span class="nf">draw_box</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ax</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">save_path</span><span class="p">,</span> <span class="n">percentiles</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">show</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+
+<div class="viewcode-block" id="ColumnWiseAnalysis.draw_box">
+<a class="viewcode-back" href="../../../data_juicer.analysis.html#data_juicer.analysis.ColumnWiseAnalysis.draw_box">[docs]</a>
+    <span class="k">def</span> <span class="nf">draw_box</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ax</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">save_path</span><span class="p">,</span> <span class="n">percentiles</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">show</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Draw the box plot for the data.</span>
 
@@ -375,7 +388,9 @@ <h1>Source code for data_juicer.analysis.column_wise_analysis</h1><div class="hi
                 <span class="c1"># if no showing, we need to clear this axes to avoid</span>
                 <span class="c1"># accumulated overlapped figures in different draw_xxx function</span>
                 <span class="c1"># calling</span>
-                <span class="n">ax</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span></div></div>
+                <span class="n">ax</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span></div>
+</div>
+
 </pre></div>
 
            </div>