deploy: 798b113

_modules/icenet/tools/process.html

@@ -535,7 +535,6 @@ <h1>Source code for icenet.tools.process</h1><div class="highlight"><pre>
 
 <span class="kn">from</span> <span class="nn">importlib</span> <span class="kn">import</span> <span class="n">import_module</span>
 <span class="kn">import</span> <span class="nn">os</span>
-<span class="kn">import</span> <span class="nn">copy</span>
 <span class="kn">import</span> <span class="nn">pickle</span>
 <span class="kn">import</span> <span class="nn">xgboost</span>
 
@@ -1195,7 +1194,7 @@ <h1>Source code for icenet.tools.process</h1><div class="highlight"><pre>
 <span class="sd">        </span>
 <span class="sd">        with open(os.path.join(cache_directory, f&#39;output_{i}.pkl&#39;), &#39;rb&#39;) as handle:</span>
 <span class="sd">            X_, Y_, W_, ids, info, genesis_args = pickle.load(handle)</span>
-<span class="sd">            </span>
+
 <span class="sd">            if i &gt; 0:</span>
 <span class="sd">                X = np.concatenate((X, X_), axis=0) # awkward will cast numpy automatically</span>
 <span class="sd">                Y = np.concatenate((Y, Y_), axis=0)</span>
@@ -1440,7 +1439,7 @@ <h1>Source code for icenet.tools.process</h1><div class="highlight"><pre>
             <span class="n">output</span><span class="p">[</span><span class="s1">&#39;trn&#39;</span><span class="p">][</span><span class="s1">&#39;data&#39;</span><span class="p">],</span> <span class="n">imputer</span> <span class="o">=</span> <span class="n">impute_datasets</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">output</span><span class="p">[</span><span class="s1">&#39;trn&#39;</span><span class="p">][</span><span class="s1">&#39;data&#39;</span><span class="p">],</span> <span class="n">features</span><span class="o">=</span><span class="n">impute_vars</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="n">args</span><span class="p">[</span><span class="s1">&#39;imputation_param&#39;</span><span class="p">],</span> <span class="n">imputer</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
             <span class="n">output</span><span class="p">[</span><span class="s1">&#39;val&#39;</span><span class="p">][</span><span class="s1">&#39;data&#39;</span><span class="p">],</span> <span class="n">imputer</span> <span class="o">=</span> <span class="n">impute_datasets</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">output</span><span class="p">[</span><span class="s1">&#39;val&#39;</span><span class="p">][</span><span class="s1">&#39;data&#39;</span><span class="p">],</span> <span class="n">features</span><span class="o">=</span><span class="n">impute_vars</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="n">args</span><span class="p">[</span><span class="s1">&#39;imputation_param&#39;</span><span class="p">],</span> <span class="n">imputer</span><span class="o">=</span><span class="n">imputer</span><span class="p">)</span>
 
-            <span class="n">fmodel</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;modeldir&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">/imputer.pkl&#39;</span>
+            <span class="n">fmodel</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;modeldir&quot;</span><span class="p">],</span> <span class="sa">f</span><span class="s1">&#39;imputer__</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;__hash_post_genesis__&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">.pkl&#39;</span><span class="p">)</span>
 
             <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Saving imputer to: </span><span class="si">{</span><span class="n">fmodel</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;green&#39;</span><span class="p">)</span>
             <span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">imputer</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="n">fmodel</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">),</span> <span class="n">protocol</span><span class="o">=</span><span class="n">pickle</span><span class="o">.</span><span class="n">HIGHEST_PROTOCOL</span><span class="p">)</span>
@@ -1451,9 +1450,9 @@ <h1>Source code for icenet.tools.process</h1><div class="highlight"><pre>
         <span class="k">if</span> <span class="n">args</span><span class="p">[</span><span class="s1">&#39;reweight&#39;</span><span class="p">]:</span>
 
             <span class="k">if</span> <span class="n">args</span><span class="p">[</span><span class="s2">&quot;reweight_file&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-                <span class="n">fmodel</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;datadir&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">/reweighter_</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;__hash_genesis__&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">.pkl&#39;</span> 
+                <span class="n">fmodel</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;datadir&quot;</span><span class="p">],</span> <span class="sa">f</span><span class="s1">&#39;reweighter__</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;__hash_post_genesis__&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">.pkl&#39;</span><span class="p">)</span> 
             <span class="k">else</span><span class="p">:</span>    
-                <span class="n">fmodel</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;datadir&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">/</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;reweight_file&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span> 
+                <span class="n">fmodel</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;datadir&quot;</span><span class="p">],</span> <span class="n">args</span><span class="p">[</span><span class="s2">&quot;reweight_file&quot;</span><span class="p">])</span> 
 
             <span class="k">if</span> <span class="s1">&#39;load&#39;</span> <span class="ow">in</span> <span class="n">args</span><span class="p">[</span><span class="s1">&#39;reweight_mode&#39;</span><span class="p">]:</span>
                 <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Loading reweighting model from: </span><span class="si">{</span><span class="n">fmodel</span><span class="si">}</span><span class="s1"> [runmode = </span><span class="si">{</span><span class="n">runmode</span><span class="si">}</span><span class="s1">]&#39;</span><span class="p">,</span> <span class="s1">&#39;green&#39;</span><span class="p">)</span>
@@ -1483,7 +1482,7 @@ <h1>Source code for icenet.tools.process</h1><div class="highlight"><pre>
         <span class="c1">## Imputate</span>
         <span class="k">if</span> <span class="n">args</span><span class="p">[</span><span class="s1">&#39;imputation_param&#39;</span><span class="p">][</span><span class="s1">&#39;active&#39;</span><span class="p">]:</span>
 
-            <span class="n">fmodel</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;modeldir&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">/imputer.pkl&#39;</span>
+            <span class="n">fmodel</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;modeldir&quot;</span><span class="p">],</span> <span class="sa">f</span><span class="s1">&#39;imputer__</span><span class="si">{</span><span class="n">args</span><span class="p">[</span><span class="s2">&quot;__hash_post_genesis__&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">.pkl&#39;</span><span class="p">)</span>
 
             <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Loading imputer from: </span><span class="si">{</span><span class="n">fmodel</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;green&#39;</span><span class="p">)</span>
             <span class="n">imputer</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">fmodel</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">))</span>