From c7d50fc5f50b80497dea03a0544fc82fd51fc9fa Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Mon, 25 Dec 2023 17:08:18 +0800 Subject: [PATCH] dev(narugo): add README.md --- .github/workflows/release.yml | 131 +++++++++++++++++++++++++++ .github/workflows/release_test.yml | 128 +++++++++++++++++++++++++++ .github/workflows/test.yml | 1 + README.md | 136 +++++++++++++++++++++++++++++ 4 files changed, 396 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b7bb057be..f349325bc0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,3 +48,134 @@ jobs: tag: ${{ github.ref }} overwrite: true file_glob: true + + release_cli: + name: CLI Build And Release + runs-on: ${{ matrix.os }} + if: "!contains(github.event.head_commit.message, 'ci skip')" + strategy: + fail-fast: false + matrix: + os: + - 'ubuntu-20.04' + - 'windows-2019' + - 'macos-11' + python-version: + - '3.8.0' + + steps: + - name: Get system version for Linux + if: ${{ contains(matrix.os, 'ubuntu') }} + shell: bash + run: | + echo "OS_NAME=Linux" >> $GITHUB_ENV + echo "IS_WIN=" >> $GITHUB_ENV + echo "IS_MAC=" >> $GITHUB_ENV + - name: Get system version for Windows + if: ${{ contains(matrix.os, 'windows') }} + shell: bash + run: | + echo "OS_NAME=Windows" >> $GITHUB_ENV + echo "IS_WIN=1" >> $GITHUB_ENV + echo "IS_MAC=" >> $GITHUB_ENV + - name: Get system version for MacOS + if: ${{ contains(matrix.os, 'macos') }} + shell: bash + run: | + echo "OS_NAME=MacOS" >> $GITHUB_ENV + echo "IS_WIN=" >> $GITHUB_ENV + echo "IS_MAC=1" >> $GITHUB_ENV + - name: Set environment for Cpython + if: ${{ !contains(matrix.python-version, 'pypy') }} + shell: bash + run: | + echo "IS_PYPY=" >> $GITHUB_ENV + - name: Set environment for PyPy + if: ${{ contains(matrix.python-version, 'pypy') }} + shell: bash + run: | + echo "IS_PYPY=1" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v3.3.0 + with: + fetch-depth: 20 + submodules: 'recursive' + - name: Set up system dependencies on Linux + if: ${{ env.OS_NAME == 'Linux' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y tree cloc wget curl make zip + - name: Set up system dependencies on Windows + if: ${{ env.OS_NAME == 'Windows' }} + shell: bash + run: | + choco install tree cloc wget curl make zip + - name: Set up system dependencies on MacOS + if: ${{ env.OS_NAME == 'MacOS' }} + run: | + brew install tree cloc wget curl make zip + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-build.txt + - name: Test the basic environment + shell: bash + run: | + python -V + pip --version + pip list + tree . + - name: Get package version + shell: bash + run: | + python -c 'from hfutils.config.meta import __VERSION__;print(__VERSION__)' + echo "PACKAGE_VERSION=$(python -c 'from hfutils.config.meta import __VERSION__;print(__VERSION__)')" >> $GITHUB_ENV + echo "GIT_COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)" >> $GITHUB_ENV + echo "PYTHON_VERSION=$(python -V | xargs -n 1 | tail -1)" >> $GITHUB_ENV + echo "CPU_ARCH=$(uname -m)" >> $GITHUB_ENV + - name: Get CLI name + shell: bash + run: | + echo "ZIP_NAME=hfutils-v${{ env.PACKAGE_VERSION }}-${{ env.OS_NAME }}-${{ env.CPU_ARCH }}-${{ env.GIT_COMMIT_ID }}-standalone" >> $GITHUB_ENV + echo "CLI_NAME=hfutils-v${{ env.PACKAGE_VERSION }}-${{ env.OS_NAME }}-${{ env.CPU_ARCH }}" >> $GITHUB_ENV + - name: Build standalone cli on Linux and MacOS + if: ${{ env.OS_NAME != 'Windows' }} + shell: bash + run: | + make build + mkdir -p dist/${{ env.CLI_NAME }} + cp dist/hfutils dist/${{ env.CLI_NAME }}/hfutils + cd dist + zip -r ${{ env.ZIP_NAME }}.zip ${{ env.CLI_NAME }} + cd .. + ls -al dist + dist/hfutils -v + dist/hfutils -h + - name: Build standalone CLI on Windows + if: ${{ env.OS_NAME == 'Windows' }} + shell: bash + run: | + make build + mkdir -p dist/${{ env.CLI_NAME }} + cp dist/hfutils.exe dist/${{ env.CLI_NAME }}/hfutils.exe + cd dist + zip -r ${{ env.ZIP_NAME }}.zip ${{ env.CLI_NAME }} + cd .. + ls -al dist + dist/hfutils -v + dist/hfutils -h + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: dist/${{ env.ZIP_NAME }}.zip + asset_name: ${{ env.ZIP_NAME }}.zip + tag: ${{ github.ref }} + overwrite: true diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml index 2ebd9c71e0..b801a9479f 100644 --- a/.github/workflows/release_test.yml +++ b/.github/workflows/release_test.yml @@ -42,3 +42,131 @@ jobs: with: name: build-artifacts-source-pack path: ./dist/* + + release_cli: + name: CLI Build And Release + runs-on: ${{ matrix.os }} + if: "!contains(github.event.head_commit.message, 'ci skip')" + strategy: + fail-fast: false + matrix: + os: + - 'ubuntu-20.04' + - 'windows-2019' + - 'macos-11' + python-version: + - '3.8.0' + + steps: + - name: Get system version for Linux + if: ${{ contains(matrix.os, 'ubuntu') }} + shell: bash + run: | + echo "OS_NAME=Linux" >> $GITHUB_ENV + echo "IS_WIN=" >> $GITHUB_ENV + echo "IS_MAC=" >> $GITHUB_ENV + - name: Get system version for Windows + if: ${{ contains(matrix.os, 'windows') }} + shell: bash + run: | + echo "OS_NAME=Windows" >> $GITHUB_ENV + echo "IS_WIN=1" >> $GITHUB_ENV + echo "IS_MAC=" >> $GITHUB_ENV + - name: Get system version for MacOS + if: ${{ contains(matrix.os, 'macos') }} + shell: bash + run: | + echo "OS_NAME=MacOS" >> $GITHUB_ENV + echo "IS_WIN=" >> $GITHUB_ENV + echo "IS_MAC=1" >> $GITHUB_ENV + - name: Set environment for Cpython + if: ${{ !contains(matrix.python-version, 'pypy') }} + shell: bash + run: | + echo "IS_PYPY=" >> $GITHUB_ENV + - name: Set environment for PyPy + if: ${{ contains(matrix.python-version, 'pypy') }} + shell: bash + run: | + echo "IS_PYPY=1" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v3.3.0 + with: + fetch-depth: 20 + submodules: 'recursive' + - name: Set up system dependencies on Linux + if: ${{ env.OS_NAME == 'Linux' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y tree cloc wget curl make zip + - name: Set up system dependencies on Windows + if: ${{ env.OS_NAME == 'Windows' }} + shell: bash + run: | + choco install tree cloc wget curl make zip + - name: Set up system dependencies on MacOS + if: ${{ env.OS_NAME == 'MacOS' }} + run: | + brew install tree cloc wget curl make zip + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-build.txt + - name: Test the basic environment + shell: bash + run: | + python -V + pip --version + pip list + tree . + - name: Get package version + shell: bash + run: | + python -c 'from hfutils.config.meta import __VERSION__;print(__VERSION__)' + echo "PACKAGE_VERSION=$(python -c 'from hfutils.config.meta import __VERSION__;print(__VERSION__)')" >> $GITHUB_ENV + echo "GIT_COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)" >> $GITHUB_ENV + echo "PYTHON_VERSION=$(python -V | xargs -n 1 | tail -1)" >> $GITHUB_ENV + echo "CPU_ARCH=$(uname -m)" >> $GITHUB_ENV + - name: Get CLI name + shell: bash + run: | + echo "ZIP_NAME=hfutils-v${{ env.PACKAGE_VERSION }}-${{ env.OS_NAME }}-${{ env.CPU_ARCH }}-${{ env.GIT_COMMIT_ID }}-standalone" >> $GITHUB_ENV + echo "CLI_NAME=hfutils-v${{ env.PACKAGE_VERSION }}-${{ env.OS_NAME }}-${{ env.CPU_ARCH }}" >> $GITHUB_ENV + - name: Build standalone cli on Linux and MacOS + if: ${{ env.OS_NAME != 'Windows' }} + shell: bash + run: | + make build + mkdir -p dist/${{ env.CLI_NAME }} + cp dist/hfutils dist/${{ env.CLI_NAME }}/hfutils + cd dist + zip -r ${{ env.ZIP_NAME }}.zip ${{ env.CLI_NAME }} + cd .. + ls -al dist + dist/hfutils -v + dist/hfutils -h + - name: Build standalone CLI on Windows + if: ${{ env.OS_NAME == 'Windows' }} + shell: bash + run: | + make build + mkdir -p dist/${{ env.CLI_NAME }} + cp dist/hfutils.exe dist/${{ env.CLI_NAME }}/hfutils.exe + cd dist + zip -r ${{ env.ZIP_NAME }}.zip ${{ env.CLI_NAME }} + cd .. + ls -al dist + dist/hfutils -v + dist/hfutils -h + - name: Upload packed files to artifacts + uses: actions/upload-artifact@v2 + with: + name: hfutils-v${{ env.PACKAGE_VERSION }}-${{ env.OS_NAME }}-${{ env.CPU_ARCH }}-${{ env.GIT_COMMIT_ID }} + path: ./dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da066b6a78..5e0a2bf8af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,7 @@ name: Code Test on: - push + - workflow_dispatch jobs: unittest: diff --git a/README.md b/README.md index b910a5e089..cbb3914862 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,139 @@ [![GitHub license](https://img.shields.io/github/license/deepghs/hfutils)](https://github.com/deepghs/hfutils/blob/master/LICENSE) Useful utilities for huggingface + +## Quick Start + +To get started with `hfutils`, you can install it using PyPI: + +```shell +pip install hfutils + +``` + +Alternatively, you can install it from the source code: + +```shell +git clone https://github.com/deepghs/hfutils.git +cd hfutils +pip install . + +``` + +Verify the installation by checking the version: + +```shell +hfutils -v + +``` + +If Python is not available in your local environment, we recommend downloading the pre-compiled executable version from +the [releases](https://github.com/deepghs/hfutils/releases). + +## Upload Data + +Upload data to repositories using the following commands: + +```shell +# Upload a single file to the repository +hfutils upload -r your/repository -i /your/local/file -f file/in/your/repo + +# Upload files in a directory as an archive file to the repository +# More formats of archive files are supported +# See: https://deepghs.github.io/hfutils/main/api_doc/archive/index.html +hfutils upload -r your/repository -i /your/local/directory -a archive/file/in/your/repo.zip + +# Upload files in a directory as a directory tree to the repository +hfutils upload -r your/repository -i /your/local/directory -d dir/in/your/repo + +``` + +You can achieve the same using the Python API: + +```python +from hfutils.operate import upload_file_to_file, upload_directory_as_archive, upload_directory_as_directory + +# Upload a single file to the repository +upload_file_to_file( + local_file='/your/local/file', + repo_id='your/repository', + file_in_repo='file/in/your/repo' +) + +# Upload files in a directory as an archive file to the repository +# More formats of archive files are supported +# See: https://deepghs.github.io/hfutils/main/api_doc/archive/index.html +upload_directory_as_archive( + local_directory='/your/local/directory', + repo_id='your/repository', + archive_in_repo='archive/file/in/your/repo.zip' +) + +# Upload files in a directory as a directory tree to the repository +upload_directory_as_directory( + local_directory='/your/local/directory', + repo_id='your/repository', + path_in_repo='dir/in/your/repo' +) +``` + +Explore additional options for uploading: + +```shell +hfutils upload -h + +``` + +## Download Data + +Download data from repositories using the following commands: + +```shell +# Download a single file from the repository +hfutils download -r your/repository -o /your/local/file -f file/in/your/repo + +# Download an archive file from the repository and extract it to the given directory +# More formats of archive files are supported +# See: https://deepghs.github.io/hfutils/main/api_doc/archive/index.html +hfutils download -r your/repository -o /your/local/directory -a archive/file/in/your/repo.zip + +# Download files from the repository as a directory tree +hfutils download -r your/repository -o /your/local/directory -d dir/in/your/repo + +``` + +Use the Python API for the same functionality: + +```python +from hfutils.operate import download_file_to_file, download_archive_as_directory, download_directory_as_directory + +# Download a single file from the repository +download_file_to_file( + local_file='/your/local/file', + repo_id='your/repository', + file_in_repo='file/in/your/repo' +) + +# Download an archive file from the repository and extract it to the given directory +# More formats of archive files are supported +# See: https://deepghs.github.io/hfutils/main/api_doc/archive/index.html +download_archive_as_directory( + local_directory='/your/local/directory', + repo_id='your/repository', + file_in_repo='archive/file/in/your/repo.zip', +) + +# Download files from the repository as a directory tree +download_directory_as_directory( + local_directory='/your/local/directory', + repo_id='your/repository', + dir_in_repo='dir/in/your/repo' +) +``` + +Explore additional options for downloading: + +```shell +hfutils download -h + +```