Merge pull request #41 from deepghs/dev/tree

dev(narugo): add tree command

docs/source/api_doc/entry/index.rst

@@ -17,6 +17,7 @@ hfutils.entry
     index_
     ls
     ls_repo
+    rollback
+    tree
     upload
     whoami
-    rollback

docs/source/api_doc/entry/ls.rst

@@ -6,10 +6,10 @@ hfutils.entry.ls
 .. automodule:: hfutils.entry.ls
 
 
-ListItemType
+FileItemType
 -------------------------
 
-.. autoenum:: ListItemType
+.. autoenum:: FileItemType
 
 
 

docs/source/api_doc/entry/tree.rst

@@ -0,0 +1,15 @@
+hfutils.entry.tree
+================================
+
+.. currentmodule:: hfutils.entry.tree
+
+.. automodule:: hfutils.entry.tree
+
+
+TreeItem
+----------------------------------
+
+.. autoclass:: TreeItem
+
+
+

docs/source/api_doc/utils/archive.rst

@@ -0,0 +1,15 @@
+hfutils.utils.archive
+=================================
+
+.. currentmodule:: hfutils.utils.archive
+
+.. automodule:: hfutils.utils.archive
+
+
+is_archive_or_compressed
+---------------------------
+
+.. autofunction:: is_archive_or_compressed
+
+
+

docs/source/api_doc/utils/data.rst

@@ -0,0 +1,15 @@
+hfutils.utils.data
+=================================
+
+.. currentmodule:: hfutils.utils.data
+
+.. automodule:: hfutils.utils.data
+
+
+is_data_file
+---------------------------
+
+.. autofunction:: is_data_file
+
+
+

docs/source/api_doc/utils/index.rst

@@ -9,11 +9,15 @@ hfutils.utils
 .. toctree::
     :maxdepth: 3
 
+    archive
     binary
+    data
     download
+    model
     number
     path
     session
     tqdm_
+    type_
     walk
 

docs/source/api_doc/utils/model.rst

@@ -0,0 +1,15 @@
+hfutils.utils.model
+=================================
+
+.. currentmodule:: hfutils.utils.model
+
+.. automodule:: hfutils.utils.model
+
+
+is_model_file
+---------------------------
+
+.. autofunction:: is_model_file
+
+
+

docs/source/api_doc/utils/tqdm_.rst

@@ -1,4 +1,4 @@
-hfutils.utils.tqdm_
+hfutils.utils.tqdm\_
 =================================
 
 .. currentmodule:: hfutils.utils.tqdm_

docs/source/api_doc/utils/type_.rst

@@ -0,0 +1,23 @@
+hfutils.utils.type\_
+=================================
+
+.. currentmodule:: hfutils.utils.type_
+
+.. automodule:: hfutils.utils.type_
+
+
+FileItemType
+--------------------------
+
+.. autoenum:: FileItemType
+    :members: render_color
+
+
+
+get_file_type
+---------------------------
+
+.. autofunction:: get_file_type
+
+
+

hfutils/entry/cli.py

@@ -5,6 +5,7 @@
 from .ls import _add_ls_subcommand
 from .ls_repo import _add_ls_repo_subcommand
 from .rollback import _add_rollback_subcommand
+from .tree import _add_tree_subcommand
 from .upload import _add_upload_subcommand
 from .whoami import _add_whoami_subcommand
 
@@ -17,6 +18,7 @@
     _add_index_subcommand,
     _add_rollback_subcommand,
     _add_clone_subcommand,
+    _add_tree_subcommand,
 ]
 
 cli = hfutilcli

hfutils/entry/download.py

@@ -7,7 +7,7 @@
 
 from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
 from ..operate import download_file_to_file, download_archive_as_directory, download_directory_as_directory
-from ..operate.base import REPO_TYPES, RepoTypeTyping
+from ..operate.base import REPO_TYPES, RepoTypeTyping, _IGNORE_PATTERN_UNSET
 from ..utils import get_requests_session
 
 
@@ -55,12 +55,15 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                   help='Just check the file size when validating the downloaded files.', show_default=True)
     @click.option('--tmpdir', 'tmpdir', type=str, default=None,
                   help='Use custom temporary Directory.', show_default=True)
+    @click.option('--all', 'show_all', is_flag=True, type=bool, default=False,
+                  help='Show all files, including hidden files.', show_default=True)
     @command_wrap()
     def download(
             repo_id: str, repo_type: RepoTypeTyping,
             file_in_repo: Optional[str], archive_in_repo: Optional[str], dir_in_repo: Optional[str],
             output_path: str, revision: str, max_workers: int,
-            password: Optional[str], wildcard: Optional[str], soft_mode_when_check: bool, tmpdir: Optional[str]
+            password: Optional[str], wildcard: Optional[str], soft_mode_when_check: bool, tmpdir: Optional[str],
+            show_all: bool = False,
     ):
         """
         Download data from HuggingFace repositories.
@@ -135,6 +138,7 @@ def download(
                 silent=False,
                 max_workers=max_workers,
                 soft_mode_when_check=soft_mode_when_check,
+                ignore_patterns=_IGNORE_PATTERN_UNSET if not show_all else [],
             )
 
         else:

hfutils/entry/ls.py

@@ -1,6 +1,5 @@
-import mimetypes
 import os.path
-from enum import Enum, unique
+import os.path
 from typing import Union, List
 
 import click
@@ -10,46 +9,7 @@
 
 from .base import CONTEXT_SETTINGS
 from ..operate.base import REPO_TYPES, get_hf_client
-from ..utils import get_requests_session
-
-mimetypes.add_type('image/webp', '.webp')
-
-
-@unique
-class ListItemType(Enum):
-    """
-    Enum class representing different types of list items.
-    """
-
-    FILE = 0x1
-    FOLDER = 0x2
-    IMAGE = 0x3
-    ARCHIVE = 0x4
-    MODEL = 0x5
-    DATA = 0x6
-
-    @property
-    def render_color(self):
-        """
-        Get the render color based on the item type.
-
-        :return: The render color for the item type.
-        :rtype: str
-        """
-        if self == self.FILE:
-            return None
-        elif self == self.FOLDER:
-            return 'blue'
-        elif self == self.IMAGE:
-            return 'magenta'
-        elif self == self.ARCHIVE:
-            return 'red'
-        elif self == self.MODEL:
-            return 'green'
-        elif self == self.DATA:
-            return 'yellow'
-        else:
-            raise ValueError(f'Unknown type - {self!r}')  # pragma: no cover
+from ..utils import get_requests_session, FileItemType, get_file_type
 
 
 class ListItem:
@@ -66,22 +26,9 @@ def __init__(self, item: Union[RepoFolder, RepoFile], base_dir: str):
         self.item = item
         self.base_dir = base_dir
         if isinstance(item, RepoFolder):
-            self.type = ListItemType.FOLDER
+            self.type = FileItemType.FOLDER
         else:
-            mimetype, _ = mimetypes.guess_type(item.path)
-            _, ext = os.path.splitext(item.path)
-            self.type = ListItemType.FILE
-            if ext in {'.ckpt', '.pt', '.safetensors', '.onnx', '.model', '.h5', '.mlmodel',
-                       '.ftz', '.pb', '.pth', '.tflite'}:
-                self.type = ListItemType.MODEL
-            elif ext in {'.json', '.csv', '.tsv', '.arrow', '.bin', '.msgpack', '.npy', '.npz',
-                         '.parquet', '.pickle', '.pkl', '.wasm'}:
-                self.type = ListItemType.DATA
-            elif mimetype:
-                if 'image' in mimetype:
-                    self.type = ListItemType.IMAGE
-                elif mimetype.split('/', maxsplit=1)[-1] in {'zip', 'rar', 'x-tar', 'x-7z-compressed'}:
-                    self.type = ListItemType.ARCHIVE
+            self.type = get_file_type(item.path)
 
 
 def _add_ls_subcommand(cli: click.Group) -> click.Group:
@@ -154,7 +101,7 @@ def ls(repo_id: str, repo_type: str, dir_in_repo, revision: str, show_all: bool,
             max_size_length = 0
             max_commit_info_length = 0
             for item in items:
-                if item.type == ListItemType.FOLDER:
+                if item.type == FileItemType.FOLDER:
                     size_text = '-'
                 else:
                     size_text = str(item.item.size)
@@ -164,8 +111,8 @@ def ls(repo_id: str, repo_type: str, dir_in_repo, revision: str, show_all: bool,
                 max_commit_info_length = max(max_commit_info_length, len(commit_text))
 
             for item in items:
-                print('d' if item.type == ListItemType.FOLDER else '-', end='')
-                print('L' if item.type != ListItemType.FOLDER and item.item.lfs else '-', end='')
+                print('d' if item.type == FileItemType.FOLDER else '-', end='')
+                print('L' if item.type != FileItemType.FOLDER and item.item.lfs else '-', end='')
 
                 commit_text = (item.item.last_commit.title or '').splitlines(keepends=False)[0]
                 print(
@@ -174,7 +121,7 @@ def ls(repo_id: str, repo_type: str, dir_in_repo, revision: str, show_all: bool,
                     end=''
                 )
 
-                if item.type == ListItemType.FOLDER:
+                if item.type == FileItemType.FOLDER:
                     size_text = '-'
                 else:
                     size_text = str(item.item.size)