Merge pull request #18 from deepghs/fix/path

dev(narugo): fix path issues

.github/workflows/ir_repos.yml

@@ -0,0 +1,116 @@
+#file: noinspection YAMLSchemaValidation
+name: Irregular Repos
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 * * * 0'
+
+jobs:
+  check_irregular_repo:
+    name: Check Repos
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - 'ubuntu-latest'
+        python-version:
+          - '3.8'
+
+    steps:
+      - name: Get system version for Linux
+        if: ${{ contains(matrix.os, 'ubuntu') }}
+        shell: bash
+        run: |
+          echo "OS_NAME=Linux" >> $GITHUB_ENV
+          echo "IS_WIN=" >> $GITHUB_ENV
+          echo "IS_MAC=" >> $GITHUB_ENV
+      - name: Get system version for Windows
+        if: ${{ contains(matrix.os, 'windows') }}
+        shell: bash
+        run: |
+          echo "OS_NAME=Windows" >> $GITHUB_ENV
+          echo "IS_WIN=1" >> $GITHUB_ENV
+          echo "IS_MAC=" >> $GITHUB_ENV
+      - name: Get system version for MacOS
+        if: ${{ contains(matrix.os, 'macos') }}
+        shell: bash
+        run: |
+          echo "OS_NAME=MacOS" >> $GITHUB_ENV
+          echo "IS_WIN=" >> $GITHUB_ENV
+          echo "IS_MAC=1" >> $GITHUB_ENV
+      - name: Set environment for Cpython
+        if: ${{ !contains(matrix.python-version, 'pypy') }}
+        shell: bash
+        run: |
+          echo "IS_PYPY=" >> $GITHUB_ENV
+      - name: Set environment for PyPy
+        if: ${{ contains(matrix.python-version, 'pypy') }}
+        shell: bash
+        run: |
+          echo "IS_PYPY=1" >> $GITHUB_ENV
+      - name: Checkout code
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 20
+          submodules: 'recursive'
+      - name: Set up system dependences on Linux
+        if: ${{ env.OS_NAME == 'Linux' }}
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tree cloc wget curl make zip
+          sudo apt-get install -y git-lfs
+          sudo apt-get install p7zip-full rar unrar
+      - name: Set up system dependences on Windows
+        if: ${{ env.OS_NAME == 'Windows' }}
+        shell: bash
+        run: |
+          choco install tree cloc wget curl make zip
+          choco install 7zip winrar  # unrar should be added
+      - name: Set up system dependences on MacOS
+        if: ${{ env.OS_NAME == 'MacOS' }}
+        run: |
+          brew install tree cloc wget curl make zip
+          brew install sevenzip
+          brew install --cask rar
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade flake8 setuptools wheel twine
+          pip install -r requirements.txt
+          pip install -r requirements-test.txt
+      - name: Test the basic environment
+        shell: bash
+        run: |
+          python -V
+          pip --version
+          pip list
+          tree .
+          cloc hfutils
+          cloc test
+      - name: Run unittest
+        env:
+          CI: 'true'
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        shell: bash
+        run: |
+          python -m tools.irregular_repo
+      - name: Change Commit
+        id: commit
+        run: |
+          git config user.name 'narugo1992'
+          git config user.email '[email protected]'
+          git add -A
+          git diff-index --quiet HEAD || git commit -a -m "dev(narugo): auto sync irregular repositories $(date -R)"
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}

.gitignore

@@ -1223,4 +1223,5 @@ fabric.properties
 *.pt
 /runs
 /YOLOv8
-.benchmarks
+.benchmarks
+!/hfutils/utils/irregular_repo.json

docs/source/api_doc/utils/index.rst

@@ -11,6 +11,7 @@ hfutils.utils
 
     binary
     download
+    path
     tqdm_
     walk
 

docs/source/api_doc/utils/path.rst

@@ -0,0 +1,36 @@
+hfutils.utils.path
+=================================
+
+.. currentmodule:: hfutils.utils.path
+
+.. automodule:: hfutils.utils.path
+
+
+hf_normpath
+--------------------------------------------
+
+.. autofunction:: hf_normpath
+
+
+
+hf_fs_path
+--------------------------------------------
+
+.. autofunction:: hf_fs_path
+
+
+
+parse_hf_fs_path
+--------------------------------------------
+
+.. autofunction:: parse_hf_fs_path
+
+
+
+HfFileSystemPath
+--------------------------------------------
+
+.. autoclass:: HfFileSystemPath
+
+
+

hfutils/operate/base.py

@@ -5,6 +5,8 @@
 
 from huggingface_hub import HfApi, HfFileSystem
 
+from hfutils.utils import parse_hf_fs_path
+
 RepoTypeTyping = Literal['dataset', 'model', 'space']
 REPO_TYPES = ['dataset', 'model', 'space']
 
@@ -113,12 +115,15 @@ def list_files_in_repository(repo_id: str, repo_type: RepoTypeTyping = 'dataset'
 
     try:
         _exist_files = [
-            os.path.relpath(file, repo_root_path)
+            parse_hf_fs_path(file).filename
             for file in hf_fs.glob(f'{repo_root_path}/**', revision=revision)
         ]
     except FileNotFoundError:
         return []
-    _exist_ps = sorted([(file, file.split(os.sep)) for file in _exist_files], key=lambda x: x[1])
+    if subdir and subdir != '.':
+        _exist_files = [os.path.relpath(file, subdir) for file in _exist_files]
+
+    _exist_ps = sorted([(file, file.split('/')) for file in _exist_files], key=lambda x: x[1])
     retval = []
     for i, (file, segments) in enumerate(_exist_ps):
         if i < len(_exist_ps) - 1 and segments == _exist_ps[i + 1][1][:len(segments)]:

hfutils/utils/__init__.py

@@ -1,6 +1,6 @@
 from .binary import is_binary_file
 from .download import download_file
-from .path import hf_normpath
+from .path import hf_normpath, hf_fs_path, parse_hf_fs_path, HfFileSystemPath
 from .temp import TemporaryDirectory
 from .tqdm_ import tqdm
 from .walk import walk_files