Update documentation

int-brain-lab · Oct 17, 2023 · 83f7081 · 83f7081
1 parent 81cb02d
commit 83f7081
Show file tree

Hide file tree

Showing 16 changed files with 268 additions and 20 deletions.
diff --git a/_autosummary/one.alf.files.html b/_autosummary/one.alf.files.html
@@ -165,13 +165,16 @@
 <tr class="row-even"><td><p><a class="reference internal" href="#one.alf.files.get_session_path" title="one.alf.files.get_session_path"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_session_path</span></code></a></p></td>
 <td><p>Returns the session path from any filepath if the date/number pattern is found, including the root directory.</p></td>
 </tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#one.alf.files.rel_path_parts" title="one.alf.files.rel_path_parts"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rel_path_parts</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#one.alf.files.padded_sequence" title="one.alf.files.padded_sequence"><code class="xref py py-obj docutils literal notranslate"><span class="pre">padded_sequence</span></code></a></p></td>
+<td><p>Ensures a file path contains a zero-padded experiment sequence folder.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#one.alf.files.rel_path_parts" title="one.alf.files.rel_path_parts"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rel_path_parts</span></code></a></p></td>
 <td><p>Parse a relative path into the relevant parts.</p></td>
 </tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#one.alf.files.remove_uuid_string" title="one.alf.files.remove_uuid_string"><code class="xref py py-obj docutils literal notranslate"><span class="pre">remove_uuid_string</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#one.alf.files.remove_uuid_string" title="one.alf.files.remove_uuid_string"><code class="xref py py-obj docutils literal notranslate"><span class="pre">remove_uuid_string</span></code></a></p></td>
 <td><p>Remove UUID from a filename of an ALF path.</p></td>
 </tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#one.alf.files.session_path_parts" title="one.alf.files.session_path_parts"><code class="xref py py-obj docutils literal notranslate"><span class="pre">session_path_parts</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#one.alf.files.session_path_parts" title="one.alf.files.session_path_parts"><code class="xref py py-obj docutils literal notranslate"><span class="pre">session_path_parts</span></code></a></p></td>
 <td><p>Parse a session path into the relevant parts.</p></td>
 </tr>
 </tbody>
@@ -499,6 +502,36 @@
 </div>
 </dd></dl>
 
+<dl class="py function">
+<dt class="sig sig-object py" id="one.alf.files.padded_sequence">
+<span class="sig-name descname"><span class="pre">padded_sequence</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">filepath</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/one/alf/files.html#padded_sequence"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.alf.files.padded_sequence" title="Link to this definition"></a></dt>
+<dd><p>Ensures a file path contains a zero-padded experiment sequence folder.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>filepath</strong> (<em>str</em><em>, </em><em>pathlib.Path</em><em>, </em><em>pathlib.PurePath</em>) – A session or file path to convert.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>The same path but with the experiment sequence folder zero-padded.  If a PurePath was
+passed, a PurePath will be returned, otherwise a Path object is returned.</p>
+</dd>
+<dt class="field-odd">Return type<span class="colon">:</span></dt>
+<dd class="field-odd"><p>pathlib.Path, pathlib.PurePath</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">filepath</span> <span class="o">=</span> <span class="s1">&#39;/iblrigdata/subject/2023-01-01/1/_ibl_experiment.description.yaml&#39;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">padded_sequence</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
+<span class="go">pathlib.Path(&#39;/iblrigdata/subject/2023-01-01/001/_ibl_experiment.description.yaml&#39;)</span>
+</pre></div>
+</div>
+<p>Supports folders and will not affect already padded paths</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">session_path</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">PurePosixPath</span><span class="p">(</span><span class="s1">&#39;subject/2023-01-01/001&#39;</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">padded_sequence</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
+<span class="go">pathlib.PurePosixPath(&#39;subject/2023-01-01/001&#39;)</span>
+</pre></div>
+</div>
+</dd></dl>
+
 </section>
 
 

diff --git a/_autosummary/one.remote.aws.html b/_autosummary/one.remote.aws.html
@@ -169,6 +169,9 @@
 <tr class="row-odd"><td><p><a class="reference internal" href="#one.remote.aws.s3_download_folder" title="one.remote.aws.s3_download_folder"><code class="xref py py-obj docutils literal notranslate"><span class="pre">s3_download_folder</span></code></a></p></td>
 <td><p>Downloads S3 folder content to a local folder.</p></td>
 </tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#one.remote.aws.url2uri" title="one.remote.aws.url2uri"><code class="xref py py-obj docutils literal notranslate"><span class="pre">url2uri</span></code></a></p></td>
+<td><p>Convert a generic Amazon virtual host URL to an S3 URI.</p></td>
+</tr>
 </tbody>
 </table>
 <dl class="py function">
@@ -192,6 +195,27 @@
 </dl>
 </dd></dl>
 
+<dl class="py function">
+<dt class="sig sig-object py" id="one.remote.aws.url2uri">
+<span class="sig-name descname"><span class="pre">url2uri</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">return_location</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/one/remote/aws.html#url2uri"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.remote.aws.url2uri" title="Link to this definition"></a></dt>
+<dd><p>Convert a generic Amazon virtual host URL to an S3 URI.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>data_path</strong> (<em>str</em>) – An Amazon virtual host URL to convert.</p></li>
+<li><p><strong>return_location</strong> (<em>bool</em>) – If true, additionally returns the location string.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p><ul class="simple">
+<li><p><em>str</em> – An S3 URI with scheme ‘s3://’.</p></li>
+<li><p><em>str</em> – If return_location is true, returns the bucket location, e.g. ‘eu-east-1’.</p></li>
+</ul>
+</p>
+</dd>
+</dl>
+</dd></dl>
+
 <dl class="py function">
 <dt class="sig sig-object py" id="one.remote.aws.is_folder">
 <span class="sig-name descname"><span class="pre">is_folder</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj_summery</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="reference internal" href="../_modules/one/remote/aws.html#is_folder"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.remote.aws.is_folder" title="Link to this definition"></a></dt>

diff --git a/_autosummary/one.tests.alf.test_alf_files.html b/_autosummary/one.tests.alf.test_alf_files.html
@@ -202,6 +202,12 @@
 <dd><p>Test for one.alf.files.remove_uuid_string.</p>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="one.tests.alf.test_alf_files.TestAlfParse.test_padded_sequence">
+<span class="sig-name descname"><span class="pre">test_padded_sequence</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/one/tests/alf/test_alf_files.html#TestAlfParse.test_padded_sequence"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.tests.alf.test_alf_files.TestAlfParse.test_padded_sequence" title="Link to this definition"></a></dt>
+<dd><p>Test for one.alf.files.padded_sequence.</p>
+</dd></dl>
+
 </dd></dl>
 
 <dl class="py class">

diff --git a/_autosummary/one.tests.remote.test_aws.html b/_autosummary/one.tests.remote.test_aws.html
@@ -221,7 +221,13 @@
 <dl class="py method">
 <dt class="sig sig-object py" id="one.tests.remote.test_aws.TestUtils.test_get_s3_virtual_host">
 <span class="sig-name descname"><span class="pre">test_get_s3_virtual_host</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/one/tests/remote/test_aws.html#TestUtils.test_get_s3_virtual_host"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.tests.remote.test_aws.TestUtils.test_get_s3_virtual_host" title="Link to this definition"></a></dt>
-<dd><p>Tests for one.remote.aws.get_s3_virtual_host function</p>
+<dd><p>Tests for one.remote.aws.get_s3_virtual_host function.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="one.tests.remote.test_aws.TestUtils.test_url2uri">
+<span class="sig-name descname"><span class="pre">test_url2uri</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/one/tests/remote/test_aws.html#TestUtils.test_url2uri"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.tests.remote.test_aws.TestUtils.test_url2uri" title="Link to this definition"></a></dt>
+<dd><p>Tests for one.remote.aws.url2uri function.</p>
 </dd></dl>
 
 </dd></dl>

diff --git a/_autosummary/one.tests.remote.test_globus.html b/_autosummary/one.tests.remote.test_globus.html
@@ -336,6 +336,12 @@
 <dd><p>Test for Globus.login and Globus.logout methods.</p>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="one.tests.remote.test_globus.TestGlobusClient.test_save_refresh_token_callback">
+<span class="sig-name descname"><span class="pre">test_save_refresh_token_callback</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/one/tests/remote/test_globus.html#TestGlobusClient.test_save_refresh_token_callback"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#one.tests.remote.test_globus.TestGlobusClient.test_save_refresh_token_callback" title="Link to this definition"></a></dt>
+<dd><p>Test for Globus._save_refresh_token_callback method.</p>
+</dd></dl>
+
 </dd></dl>
 
 <dl class="py class">

diff --git a/_modules/one/alf/files.html b/_modules/one/alf/files.html
@@ -579,6 +579,45 @@ <h1>Source code for one.alf.files</h1><div class="highlight"><pre>
         <span class="n">file_path</span> <span class="o">=</span> <span class="n">file_path</span><span class="o">.</span><span class="n">with_name</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">name_parts</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="o">+</span> <span class="n">file_path</span><span class="o">.</span><span class="n">suffix</span><span class="p">)</span>
     <span class="k">return</span> <span class="n">file_path</span></div>
 
+
+
+<div class="viewcode-block" id="padded_sequence">
+<a class="viewcode-back" href="../../../_autosummary/one.alf.files.html#one.alf.files.padded_sequence">[docs]</a>
+<span class="k">def</span> <span class="nf">padded_sequence</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Ensures a file path contains a zero-padded experiment sequence folder.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    filepath : str, pathlib.Path, pathlib.PurePath</span>
+<span class="sd">        A session or file path to convert.</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    pathlib.Path, pathlib.PurePath</span>
+<span class="sd">        The same path but with the experiment sequence folder zero-padded.  If a PurePath was</span>
+<span class="sd">        passed, a PurePath will be returned, otherwise a Path object is returned.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; filepath = &#39;/iblrigdata/subject/2023-01-01/1/_ibl_experiment.description.yaml&#39;</span>
+<span class="sd">    &gt;&gt;&gt; padded_sequence(filepath)</span>
+<span class="sd">    pathlib.Path(&#39;/iblrigdata/subject/2023-01-01/001/_ibl_experiment.description.yaml&#39;)</span>
+
+<span class="sd">    Supports folders and will not affect already padded paths</span>
+
+<span class="sd">    &gt;&gt;&gt; session_path = pathlib.PurePosixPath(&#39;subject/2023-01-01/001&#39;)</span>
+<span class="sd">    &gt;&gt;&gt; padded_sequence(filepath)</span>
+<span class="sd">    pathlib.PurePosixPath(&#39;subject/2023-01-01/001&#39;)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="n">filepath</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
+    <span class="k">if</span> <span class="p">(</span><span class="n">session_path</span> <span class="o">:=</span> <span class="n">get_session_path</span><span class="p">(</span><span class="n">filepath</span><span class="p">))</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;path must include a valid ALF session path, e.g. subject/YYYY-MM-DD/N&#39;</span><span class="p">)</span>
+    <span class="n">idx</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">filepath</span><span class="o">.</span><span class="n">parts</span><span class="p">)</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">session_path</span><span class="o">.</span><span class="n">parts</span><span class="p">)</span>
+    <span class="n">sequence</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">session_path</span><span class="o">.</span><span class="n">parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span><span class="o">.</span><span class="n">zfill</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>  <span class="c1"># zero-pad if necessary</span>
+    <span class="k">return</span> <span class="n">filepath</span><span class="o">.</span><span class="n">parents</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="n">sequence</span><span class="p">,</span> <span class="n">filepath</span><span class="o">.</span><span class="n">relative_to</span><span class="p">(</span><span class="n">session_path</span><span class="p">))</span></div>
+
 </pre></div>
 
            </div>

diff --git a/_modules/one/remote/aws.html b/_modules/one/remote/aws.html
@@ -191,6 +191,34 @@ <h1>Source code for one.remote.aws</h1><div class="highlight"><pre>
 
 
 
+<div class="viewcode-block" id="url2uri">
+<a class="viewcode-back" href="../../../_autosummary/one.remote.aws.html#one.remote.aws.url2uri">[docs]</a>
+<span class="k">def</span> <span class="nf">url2uri</span><span class="p">(</span><span class="n">data_path</span><span class="p">,</span> <span class="n">return_location</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Convert a generic Amazon virtual host URL to an S3 URI.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    data_path : str</span>
+<span class="sd">        An Amazon virtual host URL to convert.</span>
+<span class="sd">    return_location : bool</span>
+<span class="sd">        If true, additionally returns the location string.</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    str</span>
+<span class="sd">        An S3 URI with scheme &#39;s3://&#39;.</span>
+<span class="sd">    str</span>
+<span class="sd">        If return_location is true, returns the bucket location, e.g. &#39;eu-east-1&#39;.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">parsed</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">data_path</span><span class="p">)</span>
+    <span class="k">assert</span> <span class="n">parsed</span><span class="o">.</span><span class="n">netloc</span> <span class="ow">and</span> <span class="n">parsed</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">and</span> <span class="n">parsed</span><span class="o">.</span><span class="n">path</span>
+    <span class="n">bucket_name</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">loc</span><span class="p">,</span> <span class="o">*</span><span class="n">_</span> <span class="o">=</span> <span class="n">parsed</span><span class="o">.</span><span class="n">netloc</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+    <span class="n">uri</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;s3://</span><span class="si">{</span><span class="n">bucket_name</span><span class="si">}{</span><span class="n">parsed</span><span class="o">.</span><span class="n">path</span><span class="si">}</span><span class="s1">&#39;</span>
+    <span class="k">return</span> <span class="p">(</span><span class="n">uri</span><span class="p">,</span> <span class="n">loc</span><span class="p">)</span> <span class="k">if</span> <span class="n">return_location</span> <span class="k">else</span> <span class="n">uri</span></div>
+
+
+
 <div class="viewcode-block" id="is_folder">
 <a class="viewcode-back" href="../../../_autosummary/one.remote.aws.html#one.remote.aws.is_folder">[docs]</a>
 <span class="k">def</span> <span class="nf">is_folder</span><span class="p">(</span><span class="n">obj_summery</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>