v0.2.2 optimize TCP conn

example/thread.rs

@@ -0,0 +1,34 @@
+use tokio;
+use tokio_stream::StreamExt;
+
+#[tokio::main]
+async fn main() {
+    let start_time = std::time::Instant::now();
+
+    let nums: Vec<Vec<u32>> = (0..205)
+        .collect::<Vec<_>>()
+        .chunks(50)
+        .map(|chunk| chunk.to_vec())
+        .collect();
+
+    for chunk in nums{
+        let mut tasks: Vec<_> = chunk.into_iter()
+            .map(|i| tokio::spawn(async move {
+                let api = libhfd::api::tokio::ApiBuilder::new()
+                    .with_token("hf_xxxxx")
+                    .build()
+                    .unwrap();
+                let _filename = api
+                    .dataset("OpenVideo/pexels-raw".to_string())
+                    .get(format!("data/{:06}.parquet", i).as_ref())
+                    .await;
+            }))
+            .collect();
+
+        for task in tasks {
+            task.await.unwrap();
+        }
+    }
+
+    println!("Processing time: {:?}", start_time.elapsed());
+}

hfd-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hfd-cli"
-version = "0.2.1"
+version = "0.2.2"
 edition = "2021"
 
 [dependencies]

hfd-cli/src/lib.rs

@@ -5,7 +5,7 @@ use tokio::time::Duration;
 use tokio::io::{AsyncSeekExt, SeekFrom};
 use futures::StreamExt;
 
-const CHUNK_SIZE: usize = 10_000_000;
+const CHUNK_SIZE: usize = 100_000_000;
 
 #[derive(Debug)]
 pub struct HfURL {
@@ -88,7 +88,7 @@ async fn download_chunk(
     ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let client= reqwest::Client::builder()
             .default_headers(headers.clone())
-            .pool_idle_timeout(Duration::from_secs(10))
+            .pool_idle_timeout(Duration::from_secs(1))
             .build()?;
 
         let range = format!("bytes={s}-{e}");
@@ -116,6 +116,7 @@ async fn download(
 
         let client = reqwest::Client::builder()
             .default_headers(headers.clone())
+            .pool_idle_timeout(Duration::from_secs(1))
             .build()?;
 
         let response = client.get(&url).header(RANGE, "bytes=0-0").send().await?;
@@ -180,6 +181,7 @@ impl HfClient {
 
     async fn list_files(&self) -> Result<Vec<String>, Box<dyn std::error::Error>> {
         let client = reqwest::Client::builder()
+            .pool_idle_timeout(Duration::from_secs(1))
             .build()?;
         let api = self.hf_url.api();
         let response = client.get(api)