v0.3.0 add limit support

AUTOM77 · Jun 27, 2024 · 614ad73 · 614ad73
1 parent bef5cb9
commit 614ad73
Show file tree

Hide file tree

Showing 20 changed files with 65 additions and 1,670 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,7 +3,7 @@ name: ci
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   CARGO_TERM_COLOR: always
-  CARGO_LOCA: "hfd-cli/Cargo.toml"
+  CARGO_LOCA: "Cargo.toml"
   CARGO_BIN: "hfd"
 
 permissions:
@@ -15,7 +15,7 @@ on:
     branches:
       - "main"
     paths:
-      - "hfd-cli/Cargo.toml"
+      - "Cargo.toml"
       - ".github/workflows/ci.yml"
 
 jobs:

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,10 +1,26 @@
-[workspace]
-# members = ["hfd", "hfd-cli"]
-members = ["hfd-cli"]
-resolver = "2"
+
+[package]
+name = "hfd"
+version = "0.3.0"
+edition = "2021"
+
+[dependencies]
+clap = { version= "4.5.4", features=["derive"] }
+reqwest = { version = "0.12.4", default-features = false, features = ["stream", "http2", "json", "rustls-tls"] }
+tokio = { version = "1.37.0", default-features = false, features = ["rt-multi-thread", "fs"] }
+serde_json = { version = "1.0.117", default-features = false, features = ["alloc"] }
+futures = "0.3.30"
+
+[[bin]]
+name = "hfd"
+path = "src/cli.rs"
+
+[lib]
+name = "ld_"
+path = "src/lib.rs"
 
 [profile.release]
-lto = true
+lto = false
 strip = true
 panic = 'abort'
 opt-level = 3

diff --git a/README.md b/README.md
@@ -17,8 +17,10 @@
 # Download entire public hf repo
 ./hdf https://huggingface.co/deepseek-ai/DeepSeek-V2
 
-# Download gated public hf repo with token
+# Download public hf repo with limit num
+./hdf https://huggingface.co/microsoft/Florence-2-large -n 10
 
+# Download gated public hf repo with token
 ./hdf https://huggingface.co/meta-llama/Meta-Llama-3-70B -t hf_xxxxxxxxxx
 
 # Download gated public hf repo with token and save to /data/llm

diff --git a/example/auth.rs b/example/auth.rs
diff --git a/example/dataset.rs b/example/dataset.rs
diff --git a/example/hyper/Cargo.toml b/example/hyper/Cargo.toml
diff --git a/example/hyper/main.rs b/example/hyper/main.rs
diff --git a/example/mirror.rs b/example/mirror.rs
diff --git a/example/rdd/Cargo.toml b/example/rdd/Cargo.toml
diff --git a/example/rdd/src/main.rs b/example/rdd/src/main.rs
diff --git a/example/simple.rs b/example/simple.rs