From 3d21b364557b39aba153a47075ec039a1d93cc55 Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Tue, 15 Oct 2024 16:42:50 +0200
Subject: [PATCH 1/8] update dependencies

---
 Cargo.lock | 946 +++++++++++++++++++++++++++++------------------------
 Cargo.toml |  20 +-
 2 files changed, 536 insertions(+), 430 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index daf2abd983f..45685a141d9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -27,14 +27,52 @@ dependencies = [
 ]
 
 [[package]]
-name = "atty"
-version = "0.2.14"
+name = "anstream"
+version = "0.6.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
 dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -60,27 +98,15 @@ dependencies = [
 
 [[package]]
 name = "base64"
-version = "0.13.0"
+version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "bstr"
-version = "0.2.17"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
-dependencies = [
- "lazy_static",
- "memchr",
- "regex-automata",
- "serde",
-]
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
 [[package]]
 name = "bumpalo"
@@ -88,17 +114,26 @@ version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "bytes"
-version = "1.2.1"
+version = "1.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
+checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cfg-if"
@@ -107,82 +142,75 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
-name = "core-foundation"
-version = "0.9.3"
+name = "cfg_aliases"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
-name = "core-foundation-sys"
-version = "0.8.3"
+name = "colorchoice"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.12"
+version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac"
-dependencies = [
- "cfg-if",
-]
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
 [[package]]
 name = "csv"
-version = "1.1.6"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
 dependencies = [
- "bstr",
  "csv-core",
- "itoa 0.4.8",
+ "itoa",
  "ryu",
  "serde",
 ]
 
 [[package]]
 name = "csv-core"
-version = "0.1.10"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "ctrlc"
-version = "3.2.3"
+version = "3.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173"
+checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
 dependencies = [
  "nix",
- "winapi",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
-name = "encoding_rs"
-version = "0.8.31"
+name = "env_filter"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
+checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
 dependencies = [
- "cfg-if",
+ "log",
+ "regex",
 ]
 
 [[package]]
 name = "env_logger"
-version = "0.9.1"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272"
+checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
 dependencies = [
- "atty",
+ "anstream",
+ "anstyle",
+ "env_filter",
  "humantime",
  "log",
- "regex",
- "termcolor",
 ]
 
 [[package]]
@@ -203,45 +231,21 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.102",
  "synstructure",
 ]
 
-[[package]]
-name = "fastrand"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499"
-dependencies = [
- "instant",
-]
-
 [[package]]
 name = "fnv"
 version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "form_urlencoded"
-version = "1.1.0"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
 dependencies = [
  "percent-encoding",
 ]
@@ -253,40 +257,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050"
 dependencies = [
  "futures-core",
+ "futures-sink",
 ]
 
 [[package]]
 name = "futures-core"
-version = "0.3.24"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
 [[package]]
 name = "futures-io"
-version = "0.3.24"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
 [[package]]
 name = "futures-sink"
-version = "0.3.24"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
 
 [[package]]
 name = "futures-task"
-version = "0.3.24"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
 [[package]]
 name = "futures-util"
-version = "0.3.24"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
  "futures-core",
  "futures-io",
+ "futures-sink",
  "futures-task",
  "memchr",
  "pin-project-lite",
@@ -295,64 +301,59 @@ dependencies = [
 ]
 
 [[package]]
-name = "gimli"
-version = "0.26.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
-
-[[package]]
-name = "h2"
-version = "0.3.14"
+name = "getrandom"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
+ "cfg-if",
+ "libc",
+ "wasi",
 ]
 
 [[package]]
-name = "hashbrown"
-version = "0.12.3"
+name = "gimli"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
 
 [[package]]
 name = "http"
-version = "0.2.8"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
 dependencies = [
  "bytes",
  "fnv",
- "itoa 1.0.4",
+ "itoa",
 ]
 
 [[package]]
 name = "http-body"
-version = "0.4.5"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
  "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
  "pin-project-lite",
 ]
 
@@ -362,12 +363,6 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
-[[package]]
-name = "httpdate"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
-
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -376,68 +371,68 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "0.14.20"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
+checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
 dependencies = [
  "bytes",
  "futures-channel",
- "futures-core",
  "futures-util",
- "h2",
  "http",
  "http-body",
  "httparse",
- "httpdate",
- "itoa 1.0.4",
+ "itoa",
  "pin-project-lite",
- "socket2",
+ "smallvec",
  "tokio",
- "tower-service",
- "tracing",
  "want",
 ]
 
 [[package]]
-name = "hyper-tls"
-version = "0.5.0"
+name = "hyper-rustls"
+version = "0.27.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
 dependencies = [
- "bytes",
+ "futures-util",
+ "http",
  "hyper",
- "native-tls",
+ "hyper-util",
+ "rustls",
+ "rustls-pki-types",
  "tokio",
- "tokio-native-tls",
-]
-
-[[package]]
-name = "idna"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
+ "tokio-rustls",
+ "tower-service",
+ "webpki-roots",
 ]
 
 [[package]]
-name = "indexmap"
-version = "1.9.1"
+name = "hyper-util"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
+checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
 dependencies = [
- "autocfg",
- "hashbrown",
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
 ]
 
 [[package]]
-name = "instant"
-version = "0.1.12"
+name = "idna"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
 dependencies = [
- "cfg-if",
+ "unicode-bidi",
+ "unicode-normalization",
 ]
 
 [[package]]
@@ -447,10 +442,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
 
 [[package]]
-name = "itoa"
-version = "0.4.8"
+name = "is_terminal_polyfill"
+version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
 [[package]]
 name = "itoa"
@@ -467,26 +462,17 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
 [[package]]
 name = "libc"
-version = "0.2.135"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "memchr"
@@ -511,53 +497,25 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.4"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
 dependencies = [
+ "hermit-abi",
  "libc",
- "log",
  "wasi",
- "windows-sys",
-]
-
-[[package]]
-name = "native-tls"
-version = "0.2.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
-dependencies = [
- "lazy_static",
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework",
- "security-framework-sys",
- "tempfile",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "nix"
-version = "0.25.0"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
 dependencies = [
- "autocfg",
  "bitflags",
  "cfg-if",
- "libc",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
-dependencies = [
- "hermit-abi",
+ "cfg_aliases",
  "libc",
 ]
 
@@ -572,104 +530,131 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.15.0"
+version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
 [[package]]
-name = "openssl"
-version = "0.10.42"
+name = "percent-encoding"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13"
-dependencies = [
- "bitflags",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-macros",
- "openssl-sys",
-]
+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
 
 [[package]]
-name = "openssl-macros"
+name = "pin-utils"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
-name = "openssl-probe"
-version = "0.1.5"
+name = "ppv-lite86"
+version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
+dependencies = [
+ "zerocopy",
+]
 
 [[package]]
-name = "openssl-sys"
-version = "0.9.76"
+name = "proc-macro2"
+version = "1.0.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce"
+checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a"
 dependencies = [
- "autocfg",
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
+ "unicode-ident",
 ]
 
 [[package]]
-name = "percent-encoding"
-version = "2.2.0"
+name = "quinn"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
+checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684"
+dependencies = [
+ "bytes",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "socket2",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
 
 [[package]]
-name = "pin-project-lite"
-version = "0.2.9"
+name = "quinn-proto"
+version = "0.11.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
+dependencies = [
+ "bytes",
+ "rand",
+ "ring",
+ "rustc-hash",
+ "rustls",
+ "slab",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+]
 
 [[package]]
-name = "pin-utils"
-version = "0.1.0"
+name = "quinn-udp"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b"
+dependencies = [
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.59.0",
+]
 
 [[package]]
-name = "pkg-config"
-version = "0.3.25"
+name = "quote"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+dependencies = [
+ "proc-macro2",
+]
 
 [[package]]
-name = "proc-macro2"
-version = "1.0.47"
+name = "rand"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
- "unicode-ident",
+ "libc",
+ "rand_chacha",
+ "rand_core",
 ]
 
 [[package]]
-name = "quote"
-version = "1.0.21"
+name = "rand_chacha"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
- "proc-macro2",
+ "ppv-lite86",
+ "rand_core",
 ]
 
 [[package]]
-name = "redox_syscall"
-version = "0.2.16"
+name = "rand_core"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "bitflags",
+ "getrandom",
 ]
 
 [[package]]
@@ -683,62 +668,68 @@ dependencies = [
  "regex-syntax",
 ]
 
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-
 [[package]]
 name = "regex-syntax"
 version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "reqwest"
-version = "0.11.12"
+version = "0.12.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc"
+checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
 dependencies = [
  "base64",
  "bytes",
- "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
  "http",
  "http-body",
+ "http-body-util",
  "hyper",
- "hyper-tls",
+ "hyper-rustls",
+ "hyper-util",
  "ipnet",
  "js-sys",
  "log",
  "mime",
- "native-tls",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pemfile",
+ "rustls-pki-types",
  "serde",
  "serde_json",
  "serde_urlencoded",
+ "sync_wrapper",
  "tokio",
- "tokio-native-tls",
+ "tokio-rustls",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
- "winreg",
+ "webpki-roots",
+ "windows-registry",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -764,68 +755,85 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
 
 [[package]]
-name = "ryu"
-version = "1.0.11"
+name = "rustc-hash"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
 
 [[package]]
-name = "schannel"
-version = "0.1.20"
+name = "rustls"
+version = "0.23.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
+checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8"
 dependencies = [
- "lazy_static",
- "windows-sys",
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
 ]
 
 [[package]]
-name = "security-framework"
-version = "2.7.0"
+name = "rustls-pemfile"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
 dependencies = [
- "bitflags",
- "core-foundation",
- "core-foundation-sys",
- "libc",
- "security-framework-sys",
+ "rustls-pki-types",
 ]
 
 [[package]]
-name = "security-framework-sys"
-version = "2.6.1"
+name = "rustls-pki-types"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556"
+checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b"
+
+[[package]]
+name = "rustls-webpki"
+version = "0.102.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
 dependencies = [
- "core-foundation-sys",
- "libc",
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+
 [[package]]
 name = "serde"
-version = "1.0.147"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
+dependencies = [
+ "serde_derive",
+]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.147"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.87"
+version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45"
+checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
 dependencies = [
- "itoa 1.0.4",
+ "itoa",
+ "memchr",
  "ryu",
  "serde",
 ]
@@ -837,11 +845,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
 dependencies = [
  "form_urlencoded",
- "itoa 1.0.4",
+ "itoa",
  "ryu",
  "serde",
 ]
 
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "slab"
 version = "0.4.7"
@@ -851,16 +865,34 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
 [[package]]
 name = "socket2"
-version = "0.4.7"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
+checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
 dependencies = [
  "libc",
- "winapi",
+ "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
 [[package]]
 name = "syn"
 version = "1.0.102"
@@ -872,6 +904,26 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
+dependencies = [
+ "futures-core",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.12.6"
@@ -880,31 +932,28 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.102",
  "unicode-xid",
 ]
 
 [[package]]
-name = "tempfile"
-version = "3.3.0"
+name = "thiserror"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
 dependencies = [
- "cfg-if",
- "fastrand",
- "libc",
- "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "thiserror-impl",
 ]
 
 [[package]]
-name = "termcolor"
-version = "1.1.3"
+name = "thiserror-impl"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
- "winapi-util",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -924,43 +973,28 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.21.2"
+version = "1.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
+checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
 dependencies = [
- "autocfg",
+ "backtrace",
  "bytes",
  "libc",
- "memchr",
  "mio",
- "num_cpus",
  "pin-project-lite",
  "socket2",
- "winapi",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
-name = "tokio-native-tls"
-version = "0.3.0"
+name = "tokio-rustls"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
 dependencies = [
- "native-tls",
- "tokio",
-]
-
-[[package]]
-name = "tokio-util"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "pin-project-lite",
+ "rustls",
+ "rustls-pki-types",
  "tokio",
- "tracing",
 ]
 
 [[package]]
@@ -997,9 +1031,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.8"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893"
 
 [[package]]
 name = "unicode-ident"
@@ -1022,11 +1056,17 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
-version = "2.3.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
+checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -1034,10 +1074,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "vcpkg"
-version = "0.2.15"
+name = "utf8parse"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "want"
@@ -1057,26 +1097,27 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.83"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e"
 dependencies = [
  "cfg-if",
+ "once_cell",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.83"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.79",
  "wasm-bindgen-shared",
 ]
 
@@ -1094,9 +1135,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.83"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1104,22 +1145,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.83"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.79",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.83"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
 
 [[package]]
 name = "web-sys"
@@ -1132,84 +1173,149 @@ dependencies = [
 ]
 
 [[package]]
-name = "winapi"
-version = "0.3.9"
+name = "webpki-roots"
+version = "0.26.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958"
 dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
+ "rustls-pki-types",
 ]
 
 [[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
+name = "windows-registry"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
+dependencies = [
+ "windows-result",
+ "windows-strings",
+ "windows-targets",
+]
 
 [[package]]
-name = "winapi-util"
-version = "0.1.5"
+name = "windows-result"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
 dependencies = [
- "winapi",
+ "windows-targets",
 ]
 
 [[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
+name = "windows-strings"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result",
+ "windows-targets",
+]
 
 [[package]]
 name = "windows-sys"
-version = "0.36.1"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
  "windows_i686_gnu",
+ "windows_i686_gnullvm",
  "windows_i686_msvc",
  "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
  "windows_x86_64_msvc",
 ]
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.36.1"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.36.1"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.36.1"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.36.1"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.36.1"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
-name = "winreg"
-version = "0.10.1"
+name = "zerocopy"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
 dependencies = [
- "winapi",
+ "byteorder",
+ "zerocopy-derive",
 ]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
+]
+
+[[package]]
+name = "zeroize"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
diff --git a/Cargo.toml b/Cargo.toml
index e2d6fe51b5a..29007ae425d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,13 +4,13 @@ version = "0.1.0"
 authors = ["Pietro Albini <pietro@pietroalbini.org>"]
 
 [dependencies]
-failure = "0.1.5"
-reqwest = { version = "0.11.12", features = ["blocking", "json"] }
-serde = "1.0.147"
-serde_derive = "1.0.147"
-serde_json = "1.0.87"
-log = "0.4.6"
-env_logger = "0.9.1"
-csv = "1.0.5"
-ctrlc = "3.1.1"
-crossbeam-utils = "0.8.12"
+failure = "0.1.8"
+reqwest = { version = "0.12.8", features = ["blocking", "json", "rustls-tls"], default-features = false }
+serde = "1.0.210"
+serde_derive = "1.0.210"
+serde_json = "1.0.128"
+log = "0.4.22"
+env_logger = "0.11.5"
+csv = "1.3.0"
+ctrlc = "3.4.5"
+crossbeam-utils = "0.8.20"

From c27c70655aaf21dc13e66c47b572b4b2106575b3 Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Tue, 15 Oct 2024 16:46:51 +0200
Subject: [PATCH 2/8] update edition

---
 Cargo.toml        |  1 +
 src/data.rs       |  6 ++----
 src/github/api.rs |  4 ++--
 src/github/mod.rs | 10 +++++-----
 src/utils.rs      |  2 +-
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 29007ae425d..45fb3282794 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,6 @@
 [package]
 name = "rust-repos"
+edition = "2021"
 version = "0.1.0"
 authors = ["Pietro Albini <pietro@pietroalbini.org>"]
 
diff --git a/src/data.rs b/src/data.rs
index 090ba1c056c..858be0195bd 100644
--- a/src/data.rs
+++ b/src/data.rs
@@ -18,10 +18,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-use config::Config;
-use csv;
-use prelude::*;
-use serde_json;
+use crate::config::Config;
+use crate::prelude::*;
 use std::collections::HashMap;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
diff --git a/src/github/api.rs b/src/github/api.rs
index 13d7987a5dd..224addec425 100644
--- a/src/github/api.rs
+++ b/src/github/api.rs
@@ -18,8 +18,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-use config::Config;
-use prelude::*;
+use crate::config::Config;
+use crate::prelude::*;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{header, Method, StatusCode};
 use serde::{de::DeserializeOwned, Serialize};
diff --git a/src/github/mod.rs b/src/github/mod.rs
index c09a7119e88..e91fd3de3f2 100644
--- a/src/github/mod.rs
+++ b/src/github/mod.rs
@@ -20,14 +20,14 @@
 
 mod api;
 
-use config::Config;
+use crate::config::Config;
 use crossbeam_utils::thread::scope;
-use data::{Data, Repo};
-use github::api::GitHubApi;
-use prelude::*;
+use crate::data::{Data, Repo};
+use crate::github::api::GitHubApi;
+use crate::prelude::*;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, Instant};
-use utils::wrap_thread;
+use crate::utils::wrap_thread;
 
 static WANTED_LANG: &str = "Rust";
 
diff --git a/src/utils.rs b/src/utils.rs
index 074fbd9cc57..c7d40b34b49 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -18,7 +18,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-use prelude::*;
+use crate::prelude::*;
 
 pub fn log_error(err: &Error) {
     error!("{}", err);

From acdf4b239737dd365e801c874ea82e39c23c4527 Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Sun, 20 Oct 2024 13:08:08 +0200
Subject: [PATCH 3/8] Mostly rewritten everything

still need to fix:
- timeout
- csv file header
---
 .gitignore        |   1 +
 Cargo.lock        | 469 ++++++++++++++++++++++++--------------
 Cargo.toml        |  32 ++-
 src/config.rs     |  23 +-
 src/data.rs       | 180 +++++++++------
 src/github/api.rs | 565 +++++++++++++++++++++-------------------------
 src/github/mod.rs | 211 +++++++++--------
 src/main.rs       | 145 +++---------
 src/prelude.rs    |  21 --
 src/utils.rs      |  34 ---
 10 files changed, 828 insertions(+), 853 deletions(-)
 delete mode 100644 src/prelude.rs
 delete mode 100644 src/utils.rs

diff --git a/.gitignore b/.gitignore
index 53eaa21960d..37c8717f9a0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 /target
 **/*.rs.bk
+.env
diff --git a/Cargo.lock b/Cargo.lock
index 45685a141d9..88242eeef3c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -19,62 +19,13 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.19"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
 dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "anstream"
-version = "0.6.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
-
-[[package]]
-name = "anstyle-parse"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
-dependencies = [
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
-dependencies = [
- "anstyle",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -142,22 +93,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
-name = "cfg_aliases"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
-
-[[package]]
-name = "colorchoice"
-version = "1.0.2"
+name = "color-eyre"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5"
+dependencies = [
+ "backtrace",
+ "color-spantrace",
+ "eyre",
+ "indenter",
+ "once_cell",
+ "owo-colors",
+ "tracing-error",
+]
 
 [[package]]
-name = "crossbeam-utils"
-version = "0.8.20"
+name = "color-spantrace"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2"
+dependencies = [
+ "once_cell",
+ "owo-colors",
+ "tracing-core",
+ "tracing-error",
+]
 
 [[package]]
 name = "csv"
@@ -181,58 +141,40 @@ dependencies = [
 ]
 
 [[package]]
-name = "ctrlc"
-version = "3.4.5"
+name = "dotenvy"
+version = "0.15.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
-dependencies = [
- "nix",
- "windows-sys 0.59.0",
-]
+checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
 
 [[package]]
-name = "env_filter"
-version = "0.1.2"
+name = "enum-map"
+version = "2.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9"
 dependencies = [
- "log",
- "regex",
-]
-
-[[package]]
-name = "env_logger"
-version = "0.11.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
-dependencies = [
- "anstream",
- "anstyle",
- "env_filter",
- "humantime",
- "log",
+ "enum-map-derive",
+ "serde",
 ]
 
 [[package]]
-name = "failure"
-version = "0.1.8"
+name = "enum-map-derive"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
+checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb"
 dependencies = [
- "backtrace",
- "failure_derive",
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
-name = "failure_derive"
-version = "0.1.8"
+name = "eyre"
+version = "0.6.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
+checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.102",
- "synstructure",
+ "indenter",
+ "once_cell",
 ]
 
 [[package]]
@@ -363,12 +305,6 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
-[[package]]
-name = "humantime"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
-
 [[package]]
 name = "hyper"
 version = "1.4.1"
@@ -436,16 +372,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "ipnet"
-version = "2.5.0"
+name = "indenter"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
+checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
 
 [[package]]
-name = "is_terminal_polyfill"
-version = "1.70.1"
+name = "ipnet"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
 
 [[package]]
 name = "itoa"
@@ -462,18 +398,43 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "libc"
 version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
+[[package]]
+name = "matchers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+dependencies = [
+ "regex-automata 0.1.10",
+]
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -508,15 +469,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "nix"
-version = "0.29.0"
+name = "nu-ansi-term"
+version = "0.46.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
 dependencies = [
- "bitflags",
- "cfg-if",
- "cfg_aliases",
- "libc",
+ "overload",
+ "winapi",
 ]
 
 [[package]]
@@ -534,6 +493,41 @@ version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
+[[package]]
+name = "overload"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+
+[[package]]
+name = "owo-colors"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
+
+[[package]]
+name = "parking_lot"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.1"
@@ -657,22 +651,58 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "regex"
-version = "1.6.0"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
+checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-automata 0.3.7",
+ "regex-syntax 0.7.5",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+dependencies = [
+ "regex-syntax 0.6.29",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.7.5",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
 [[package]]
 name = "regex-syntax"
-version = "0.6.27"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
+checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
 
 [[package]]
 name = "reqwest"
@@ -734,18 +764,20 @@ dependencies = [
 
 [[package]]
 name = "rust-repos"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
- "crossbeam-utils",
+ "color-eyre",
  "csv",
- "ctrlc",
- "env_logger",
- "failure",
- "log",
+ "dotenvy",
+ "enum-map",
  "reqwest",
  "serde",
  "serde_derive",
  "serde_json",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -806,6 +838,12 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
 
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "serde"
 version = "1.0.210"
@@ -823,7 +861,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn",
 ]
 
 [[package]]
@@ -850,12 +888,30 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "sharded-slab"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
+dependencies = [
+ "lazy_static",
+]
+
 [[package]]
 name = "shlex"
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.7"
@@ -893,17 +949,6 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
-[[package]]
-name = "syn"
-version = "1.0.102"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
 [[package]]
 name = "syn"
 version = "2.0.79"
@@ -924,18 +969,6 @@ dependencies = [
  "futures-core",
 ]
 
-[[package]]
-name = "synstructure"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.102",
- "unicode-xid",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.64"
@@ -953,7 +986,17 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
+dependencies = [
+ "cfg-if",
+ "once_cell",
 ]
 
 [[package]]
@@ -981,11 +1024,26 @@ dependencies = [
  "bytes",
  "libc",
  "mio",
+ "parking_lot",
  "pin-project-lite",
+ "signal-hook-registry",
  "socket2",
+ "tokio-macros",
+ "tracing",
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "tokio-macros"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.26.0"
@@ -1005,22 +1063,73 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.37"
+version = "0.1.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
 dependencies = [
- "cfg-if",
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-attributes"
+version = "0.1.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tracing-core"
-version = "0.1.30"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+dependencies = [
+ "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-error"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
+dependencies = [
+ "tracing",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
 dependencies = [
+ "log",
  "once_cell",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+dependencies = [
+ "matchers",
+ "nu-ansi-term",
+ "once_cell",
+ "regex",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
 ]
 
 [[package]]
@@ -1050,12 +1159,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-xid"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -1074,10 +1177,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "utf8parse"
-version = "0.2.2"
+name = "valuable"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
 [[package]]
 name = "want"
@@ -1117,7 +1220,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn",
  "wasm-bindgen-shared",
 ]
 
@@ -1151,7 +1254,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -1181,6 +1284,28 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-registry"
 version = "0.2.0"
@@ -1311,7 +1436,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 45fb3282794..84e48c8e65e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,17 +1,27 @@
 [package]
 name = "rust-repos"
 edition = "2021"
-version = "0.1.0"
-authors = ["Pietro Albini <pietro@pietroalbini.org>"]
+version = "0.2.0"
+authors = [
+    "Pietro Albini <pietro@pietroalbini.org>",
+    "Vivian Roest <vivian@0x76.dev>",
+]
 
 [dependencies]
-failure = "0.1.8"
-reqwest = { version = "0.12.8", features = ["blocking", "json", "rustls-tls"], default-features = false }
-serde = "1.0.210"
-serde_derive = "1.0.210"
-serde_json = "1.0.128"
-log = "0.4.22"
-env_logger = "0.11.5"
+reqwest = { version = "0.12.8", features = [
+    "blocking",
+    "json",
+    "rustls-tls",
+], default-features = false }
+serde = "1"
+serde_derive = "1"
+serde_json = "1"
 csv = "1.3.0"
-ctrlc = "3.4.5"
-crossbeam-utils = "0.8.20"
+
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tokio = { version = "1.40", features = ["full", "tracing"] }
+dotenvy = "0.15.7"
+color-eyre = "0.6.3"
+tracing = "0.1.40"
+enum-map = { version = "2.7.3", features = ["serde"] }
+thiserror = "1"
diff --git a/src/config.rs b/src/config.rs
index 9bac22dad2d..ee65bc977b2 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,27 +1,8 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
 use std::path::PathBuf;
 
+#[derive(Debug, Clone)]
 pub struct Config {
-    pub github_token: String,
+    pub github_token: Vec<String>,
     pub data_dir: PathBuf,
     pub timeout: Option<u64>,
 }
diff --git a/src/data.rs b/src/data.rs
index 858be0195bd..16cbf18c761 100644
--- a/src/data.rs
+++ b/src/data.rs
@@ -1,39 +1,29 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+use enum_map::{Enum, EnumMap};
+use serde_derive::{Deserialize, Serialize};
+use tokio::sync::Mutex;
+use tokio::task::{spawn_blocking, JoinSet};
+use tracing::{debug, info, trace};
 
 use crate::config::Config;
-use crate::prelude::*;
-use std::collections::HashMap;
+use std::collections::BTreeMap;
+use std::fs::OpenOptions;
 use std::path::PathBuf;
-use std::sync::{Arc, Mutex};
+use std::sync::atomic::AtomicUsize;
+use std::sync::Arc;
 use std::{
-    fs::{self, File, OpenOptions},
+    fs::{self, File},
     io::{prelude::*, BufWriter},
 };
 
-#[derive(Default, Serialize, Deserialize)]
-struct State {
-    last_id: HashMap<String, usize>,
+#[derive(Debug, Enum, Serialize, Deserialize, Copy, Clone)]
+pub enum Forge {
+    Github,
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize)]
+struct State(EnumMap<Forge, AtomicUsize>);
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Repo {
     pub id: String,
     pub name: String,
@@ -41,76 +31,124 @@ pub struct Repo {
     pub has_cargo_lock: bool,
 }
 
+#[derive(Debug, Clone)]
 pub struct Data {
-    base_dir: PathBuf,
+    data_dir: PathBuf,
+
+    state_lock: Arc<Mutex<()>>,
 
-    csv_write_lock: Arc<Mutex<()>>,
+    state_cache: Arc<State>,
 
-    state_path: PathBuf,
-    state_cache: Arc<Mutex<Option<State>>>,
+    repos_state: Arc<Mutex<EnumMap<Forge, BTreeMap<String, Repo>>>>,
 }
 
 impl Data {
-    pub fn new(config: &Config) -> Self {
-        Data {
-            base_dir: config.data_dir.clone(),
+    pub fn new(config: &Config) -> color_eyre::Result<Self> {
+        fs::create_dir_all(&config.data_dir)?;
 
-            csv_write_lock: Arc::new(Mutex::new(())),
+        let mut data = Data {
+            data_dir: config.data_dir.clone(),
 
-            state_path: config.data_dir.join("state.json"),
-            state_cache: Arc::new(Mutex::new(None)),
-        }
-    }
+            state_lock: Arc::new(Mutex::new(())),
+            state_cache: Arc::new(State::default()),
+            repos_state: Arc::new(Mutex::new(EnumMap::default())),
+        };
 
-    fn edit_state<T, F: Fn(&mut State) -> Fallible<T>>(&self, f: F) -> Fallible<T> {
-        let mut state_cache = self.state_cache.lock().unwrap();
+        let state_path = data.state_path();
+        if state_path.exists() {
+            let state_cache: State = serde_json::from_slice(&fs::read(&state_path)?)?;
 
-        if state_cache.is_none() {
-            if self.state_path.exists() {
-                *state_cache = Some(serde_json::from_slice(&fs::read(&self.state_path)?)?);
-            } else {
-                *state_cache = Some(Default::default());
-            }
+            data.state_cache = Arc::new(state_cache)
         }
 
-        let state = state_cache.as_mut().unwrap();
-        let result = f(state)?;
+        Ok(data)
+    }
 
-        let mut file = BufWriter::new(File::create(&self.state_path)?);
-        serde_json::to_writer_pretty(&mut file, &state)?;
-        file.write_all(&[b'\n'])?;
+    pub fn state_path(&self) -> PathBuf {
+        self.data_dir.join("state.json")
+    }
 
-        Ok(result)
+    pub fn csv_path(&self, forge: Forge) -> PathBuf {
+        match forge {
+            Forge::Github => self.data_dir.join("github.csv"),
+        }
     }
 
-    pub fn get_last_id(&self, platform: &str) -> Fallible<Option<usize>> {
-        self.edit_state(|state| Ok(state.last_id.get(platform).cloned()))
+    pub fn get_last_id(&self, forge: Forge) -> usize {
+        self.state_cache.0[forge].load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn set_last_id(&self, platform: &str, id: usize) -> Fallible<()> {
-        self.edit_state(|state| {
-            state.last_id.insert(platform.to_string(), id);
+    /// Store the state cache to disk, i.e. last fetched ids
+    async fn store_state_cache(&self) -> color_eyre::Result<()> {
+        let state = self.state_cache.clone();
+        let lock = self.state_lock.clone();
+        let state_path = self.state_path();
+        spawn_blocking(move || -> color_eyre::Result<()> {
+            let guard = lock.blocking_lock();
+
+            let file = File::create(state_path)?;
+            let mut file = BufWriter::new(file);
+            serde_json::to_writer_pretty(&mut file, state.as_ref())?;
+            file.write_all(b"\n")?;
+
+            drop(guard);
+
             Ok(())
         })
+        .await
+        .unwrap()
     }
 
-    pub fn store_repo(&self, platform: &str, repo: Repo) -> Fallible<()> {
-        // Ensure only one thread can write to CSV files at once
-        let _lock = self.csv_write_lock.lock().unwrap();
+    /// Stores the repos found to disk in a CSV
+    async fn store_csv(&self) -> color_eyre::Result<()> {
+        debug!("storing csv file");
+        let mut repos = self.repos_state.lock().await;
 
-        let file = self.base_dir.join(format!("{}.csv", platform));
+        let mut js = JoinSet::new();
 
-        // Create the new file or append to it
-        let mut csv = if file.exists() {
-            csv::WriterBuilder::new()
-                .has_headers(false)
-                .from_writer(OpenOptions::new().append(true).open(&file)?)
-        } else {
-            csv::WriterBuilder::new().from_path(&file)?
-        };
+        for (forge, repos) in repos.iter() {
+            let path = self.csv_path(forge);
+            let repos = repos.clone(); // is this necessary?
+            js.spawn_blocking(|| -> color_eyre::Result<()> {
+                let mut write_headers = false;
+                if !path.exists() {
+                    File::create(&path)?;
+                    write_headers = true;
+                }
+
+                let file = OpenOptions::new().append(true).open(path)?;
 
-        csv.serialize(repo)?;
+                let mut writer = csv::WriterBuilder::new()
+                    .has_headers(write_headers)
+                    .from_writer(file);
+
+                for (_, repo) in repos {
+                    writer.serialize(repo)?;
+                }
+
+                Ok(())
+            });
+        }
+
+        js.join_all().await.into_iter().collect::<Result<(), _>>()?;
+
+        // Clear the map
+        repos.iter_mut().for_each(|(_, m)| m.clear());
+
+        Ok(())
+    }
+
+    pub async fn set_last_id(&self, forge: Forge, n: usize) -> color_eyre::Result<()> {
+        self.state_cache.0[forge].store(n, std::sync::atomic::Ordering::SeqCst);
+
+        self.store_csv().await?;
+        self.store_state_cache().await?;
 
         Ok(())
     }
+
+    pub async fn store_repo(&self, forge: Forge, repo: Repo) {
+        let mut repos_state = self.repos_state.lock().await;
+        repos_state[forge].insert(repo.name.clone(), repo);
+    }
 }
diff --git a/src/github/api.rs b/src/github/api.rs
index 224addec425..09e87969f8a 100644
--- a/src/github/api.rs
+++ b/src/github/api.rs
@@ -1,318 +1,48 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-use crate::config::Config;
-use crate::prelude::*;
-use reqwest::blocking::{Client, RequestBuilder, Response};
-use reqwest::{header, Method, StatusCode};
-use serde::{de::DeserializeOwned, Serialize};
-use std::borrow::Cow;
-use std::sync::{
-    atomic::{AtomicBool, AtomicUsize, Ordering},
-    Arc,
+use std::{
+    borrow::Cow,
+    future::Future,
+    sync::atomic::{AtomicUsize, Ordering},
+    time::Duration,
 };
-use std::time::Duration;
 
-static USER_AGENT: &str = "rust-repos (https://github.com/rust-ops/rust-repos)";
+use reqwest::{header, Client, Method, RequestBuilder, Response, StatusCode};
+use serde::de::DeserializeOwned;
+use serde::ser::Serialize;
+use serde_derive::Deserialize;
+use serde_json::json;
+use thiserror::Error;
+use tokio::{task::yield_now, time::sleep};
+use tracing::{debug, error, warn};
 
-static GRAPHQL_QUERY_REPOSITORIES: &str = "
-query($ids: [ID!]!) {
-    nodes(ids: $ids) {
-        ... on Repository {
-            id
-            nameWithOwner
-            defaultBranchRef {
-                name
-            }
-            languages(first: 100, orderBy: { field: SIZE, direction: DESC }) {
-                nodes {
-                    name
-                }
-            }
-        }
-    }
-
-    rateLimit {
-        cost
-    }
-}
-";
-
-#[derive(Fail, Debug)]
-#[fail(display = "internal github error: {:?}", _0)]
-struct RetryRequest(StatusCode);
-
-trait ResponseExt {
-    fn handle_errors(self) -> Fallible<Self>
-    where
-        Self: Sized;
-}
+use crate::{config::Config, data::Repo};
 
-impl ResponseExt for Response {
-    fn handle_errors(self) -> Fallible<Self> {
-        let status = self.status();
-        match status {
-            StatusCode::INTERNAL_SERVER_ERROR
-            | StatusCode::BAD_GATEWAY
-            | StatusCode::SERVICE_UNAVAILABLE
-            | StatusCode::GATEWAY_TIMEOUT => Err(RetryRequest(status).into()),
-            _ => Ok(self),
-        }
-    }
-}
+static USER_AGENT: &str = "rust-repos (https://github.com/rust-lang/rust-repos)";
 
-pub struct GitHubApi<'conf> {
-    config: &'conf Config,
+#[derive(Debug)]
+pub struct Github {
     client: Client,
-    slow_down: Arc<AtomicBool>,
-    concurrent_requests: Arc<AtomicUsize>,
-}
-
-impl<'conf> GitHubApi<'conf> {
-    pub fn new(config: &'conf Config) -> Self {
-        GitHubApi {
-            config,
-            client: Client::new(),
-            slow_down: Arc::new(AtomicBool::new(false)),
-            concurrent_requests: Arc::new(AtomicUsize::new(0)),
-        }
-    }
-
-    fn retry<T, F: Fn() -> Fallible<T>>(&self, f: F) -> Fallible<T> {
-        let mut wait = Duration::from_secs(10);
-        let mut first = true;
-
-        loop {
-            let concurrent = self.concurrent_requests.fetch_add(1, Ordering::SeqCst);
-            debug!(
-                "currently making {} concurrent requests to the GitHub API",
-                concurrent + 1
-            );
-            let res = f();
-            self.concurrent_requests.fetch_sub(1, Ordering::SeqCst);
-
-            match res {
-                Ok(res) => return Ok(res),
-                Err(err) => {
-                    let mut retry = false;
-                    if let Some(error) = err.downcast_ref::<RetryRequest>() {
-                        warn!(
-                            "API call to GitHub returned status code {}, retrying in {} seconds",
-                            error.0,
-                            wait.as_secs()
-                        );
-                        retry = true;
-                    } else if let Some(error) = err.downcast_ref::<reqwest::Error>() {
-                        if error.is_timeout() {
-                            warn!(
-                                "API call to GitHub timed out, retrying in {} seconds",
-                                wait.as_secs()
-                            );
-                            retry = true;
-                        }
-                    } else if let Some(error) = err.downcast_ref::<std::io::Error>() {
-                        if error.kind() == std::io::ErrorKind::ConnectionReset {
-                            warn!(
-                                "connection to the API reset by peer, retrying in {} seconds",
-                                wait.as_secs()
-                            );
-                            retry = true;
-                        }
-                    }
-
-                    if !retry {
-                        return Err(err);
-                    }
-                }
-            }
-
-            // Slow down only once per API call
-            if first {
-                self.slow_down.store(true, Ordering::SeqCst);
-            }
-
-            ::std::thread::sleep(wait);
-
-            // Stop doubling the time after a few increments, to avoid waiting too long
-            // This is still a request every ~10 minutes
-            if wait.as_secs() < 640 {
-                wait *= 2;
-            }
-
-            first = false;
-        }
-    }
-
-    fn build_request(&self, method: Method, url: &str) -> RequestBuilder {
-        let url = if !url.starts_with("https://") {
-            Cow::Owned(format!("https://api.github.com/{}", url))
-        } else {
-            Cow::Borrowed(url)
-        };
-
-        self.client
-            .request(method, url.as_ref())
-            .header(
-                header::AUTHORIZATION,
-                format!("token {}", self.config.github_token),
-            )
-            .header(header::USER_AGENT, USER_AGENT)
-    }
-
-    fn graphql<T: DeserializeOwned, V: Serialize>(&self, query: &str, variables: V) -> Fallible<T> {
-        self.retry(|| {
-            let resp: GraphResponse<T> = self
-                .build_request(Method::POST, "graphql")
-                .json(&json!({
-                    "query": query,
-                    "variables": variables,
-                }))
-                .send()?
-                .handle_errors()?
-                .json()?;
-
-            if let Some(data) = resp.data {
-                if let Some(errors) = resp.errors {
-                    for error in errors {
-                        if let Some(ref type_) = error.type_ {
-                            if type_ == "NOT_FOUND" {
-                                debug!("ignored GraphQL error: {}", error.message);
-                                continue;
-                            }
-                        }
-
-                        warn!("non-fatal GraphQL error: {}", error.message);
-                    }
-                }
-
-                Ok(data)
-            } else if let Some(mut errors) = resp.errors {
-                Err(err_msg(errors.pop().unwrap().message)
-                    .context("GitHub GraphQL call failed")
-                    .into())
-            } else if let Some(message) = resp.message {
-                if message.contains("abuse") {
-                    warn!("triggered GitHub abuse detection systems");
-                    Err(RetryRequest(StatusCode::TOO_MANY_REQUESTS).into())
-                } else {
-                    Err(err_msg(message)
-                        .context("GitHub GraphQL call failed")
-                        .into())
-                }
-            } else {
-                Err(err_msg("empty GraphQL response"))
-            }
-        })
-    }
-
-    pub fn scrape_repositories(&self, since: usize) -> Fallible<Vec<Option<RestRepository>>> {
-        self.retry(|| {
-            let resp = self
-                .build_request(Method::GET, &format!("repositories?since={}", since))
-                .send()?
-                .handle_errors()?;
-
-            let status = resp.status();
-            if status == StatusCode::OK {
-                Ok(resp.json()?)
-            } else {
-                let error: GitHubError = resp.json()?;
-                if error.message.contains("abuse") {
-                    warn!("triggered GitHub abuse detection systems");
-                    Err(RetryRequest(StatusCode::TOO_MANY_REQUESTS).into())
-                } else {
-                    Err(err_msg(error.message)
-                        .context(format!(
-                            "GitHub API call failed with status code: {}",
-                            status
-                        ))
-                        .context(format!(
-                            "failed to fetch GitHub repositories since ID {}",
-                            since
-                        ))
-                        .into())
-                }
-            }
-        })
-    }
-
-    pub fn load_repositories(&self, node_ids: &[String]) -> Fallible<Vec<Option<GraphRepository>>> {
-        let data: GraphRepositories = self.graphql(
-            GRAPHQL_QUERY_REPOSITORIES,
-            json!({
-                "ids": node_ids,
-            }),
-        )?;
-
-        assert!(
-            data.rate_limit.cost <= 1,
-            "load repositories query too costly"
-        );
-        Ok(data.nodes)
-    }
-
-    pub fn file_exists(&self, repo: &GraphRepository, path: &str) -> Fallible<bool> {
-        let url = format!(
-            "https://raw.githubusercontent.com/{}/{}/{}",
-            repo.name_with_owner,
-            if let Some(ref_) = &repo.default_branch_ref {
-                &ref_.name
-            } else {
-                "master"
-            },
-            path,
-        );
-
-        self.retry(|| {
-            let resp = self
-                .build_request(Method::GET, &url)
-                .send()?
-                .handle_errors()?;
-            match resp.status() {
-                StatusCode::OK => Ok(true),
-                StatusCode::NOT_FOUND => Ok(false),
-                status => Err(
-                    err_msg(format!("GitHub API returned status code {}", status))
-                        .context(format!(
-                            "failed to fetch file {} from repo {}",
-                            path, repo.name_with_owner,
-                        ))
-                        .into(),
-                ),
-            }
-        })
-    }
-
-    pub fn should_slow_down(&self) -> bool {
-        self.slow_down.swap(false, Ordering::SeqCst)
-    }
+    config: Config,
+    current_token_index: AtomicUsize,
 }
 
 #[derive(Deserialize)]
-struct GitHubError {
+pub struct GitHubError {
     message: String,
     #[serde(rename = "type")]
     type_: Option<String>,
 }
 
-#[derive(Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
+pub struct Node {
+    pub path: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct GithubTree {
+    pub tree: Vec<Node>,
+}
+
+#[derive(Debug, Deserialize)]
 pub struct RestRepository {
     pub id: usize,
     pub full_name: String,
@@ -344,10 +74,20 @@ struct GraphRepositories {
 pub struct GraphRepository {
     pub id: String,
     pub name_with_owner: String,
-    pub default_branch_ref: Option<GraphRef>,
     pub languages: GraphLanguages,
 }
 
+impl GraphRepository {
+    pub fn to_repo(self, has_cargo_toml: bool, has_cargo_lock: bool) -> Repo {
+        Repo {
+            id: self.id,
+            name: self.name_with_owner,
+            has_cargo_toml,
+            has_cargo_lock,
+        }
+    }
+}
+
 #[derive(Debug, Deserialize)]
 pub struct GraphLanguages {
     pub nodes: Vec<Option<GraphLanguage>>,
@@ -363,9 +103,220 @@ pub struct GraphRef {
     pub name: String,
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
-pub enum GitHubErrorType {
-    NotFound,
-    Other(String),
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("reqwest error occurred {0:?}")]
+    Reqwest(#[from] reqwest::Error),
+    #[error("rate limit hit {0}")]
+    RateLimit(StatusCode),
+    #[error("other http error: {0}")]
+    HttpError(StatusCode),
+
+    #[error("Response did not contain requested data")]
+    EmptyData,
+    #[error("IO Error {0}")]
+    Io(#[from] std::io::Error),
+}
+
+const GRAPHQL_QUERY_REPOSITORIES: &str = "
+query($ids: [ID!]!) {
+    nodes(ids: $ids) {
+        ... on Repository {
+            id
+            nameWithOwner
+            languages(first: 100, orderBy: { field: SIZE, direction: DESC }) {
+                nodes {
+                    name
+                }
+            }
+        }
+    }
+
+    rateLimit {
+        cost
+    }
+}
+";
+
+impl Github {
+    pub fn new(config: Config) -> Self {
+        Github {
+            client: Client::new(),
+            current_token_index: AtomicUsize::new(0),
+            config,
+        }
+    }
+
+    #[inline]
+    fn get_token(&self) -> &str {
+        &self.config.github_token[self.current_token_index.load(Ordering::Relaxed)]
+    }
+
+    fn build_request(&self, method: Method, url: &str) -> RequestBuilder {
+        let url = if !url.starts_with("https://") {
+            Cow::from(format!("https://api.github.com/{}", url))
+        } else {
+            Cow::from(url)
+        };
+        debug!("Sending request to {url}");
+        self.client
+            .request(method, url.as_ref())
+            .header(header::AUTHORIZATION, format!("token {}", self.get_token()))
+            .header(header::USER_AGENT, USER_AGENT)
+        // .header(header::ACCEPT, "application/vnd.github+json")
+    }
+
+    async fn graphql<T: DeserializeOwned, S: Serialize>(
+        &self,
+        query: &str,
+        variables: S,
+    ) -> Result<T, Error> {
+        let resp = self
+            .build_request(Method::POST, "graphql")
+            .json(&json!({
+                "query": query,
+                "variables": variables,
+            }))
+            .send()
+            .await?;
+
+        let data: GraphResponse<T> = handle_response_json(resp).await?;
+
+        data.data.ok_or_else(|| Error::EmptyData)
+    }
+
+    pub async fn load_repositories(
+        &self,
+        node_ids: &[String],
+    ) -> Result<Vec<GraphRepository>, Error> {
+        let data: GraphRepositories = self
+            .retry(|| async {
+                self.graphql(
+                    GRAPHQL_QUERY_REPOSITORIES,
+                    json!({
+                        "ids": node_ids,
+                    }),
+                )
+                .await
+            })
+            .await?;
+
+        assert!(
+            data.rate_limit.cost <= 1,
+            "load repositories query too costly"
+        );
+
+        Ok(data.nodes.into_iter().flatten().collect())
+    }
+
+    /// gets a file tree of a specific github repo
+    pub async fn tree(&self, repo: &Repo, recursive: bool) -> Result<GithubTree, Error> {
+        let mut url = format!("repos/{}/git/trees/HEAD", repo.name);
+        if recursive {
+            url = format!("{url}?recursive=1")
+        }
+
+        self.retry(|| async {
+            let resp = self.build_request(Method::GET, &url).send().await?;
+
+            handle_response_json(resp).await
+        })
+        .await
+    }
+
+    /// scrapes all github repos (paginated)
+    pub async fn scrape_repositories(&self, since: usize) -> Result<Vec<RestRepository>, Error> {
+        // Maybe needs to be a Vec<Option<RestRepository>>
+        let output: Vec<RestRepository> = self
+            .retry(|| async {
+                let resp = self
+                    .build_request(Method::GET, &format!("repositories?since={}", since))
+                    .send()
+                    .await?;
+
+                handle_response_json(resp).await
+            })
+            .await?;
+
+        Ok(output)
+    }
+
+    /// retry a github api request and rotate tokens to circumvent rate limiting
+    /// On reqwest errors does exponential backoff until 5 mins.
+    async fn retry<F, Fu, R>(&self, fun: F) -> Result<R, Error>
+    where
+        F: Fn() -> Fu,
+        Fu: Future<Output = Result<R, Error>>,
+    {
+        let mut backoff = Duration::from_secs(1);
+        loop {
+            match fun().await {
+                ok @ Ok(_) => return ok,
+                Err(Error::Reqwest(reqwest_error)) => {
+                    warn!("Reqwest encountered error {reqwest_error:?}");
+                    warn!("Backing off for {} seconds", backoff.as_secs());
+                    sleep(backoff).await;
+
+                    backoff = backoff + backoff + Duration::from_millis(123); // Exponential backoff + jitter
+
+                    // After 5 minutes bail
+                    if backoff.as_secs() > 300 {
+                        error!("Failed sending request 5 times");
+                        return Err(Error::Reqwest(reqwest_error));
+                    }
+                }
+                Err(err @ Error::HttpError(_)) => return Err(err),
+                Err(Error::RateLimit(_)) => {
+                    let mut wait = false;
+                    self.current_token_index
+                        .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |old| {
+                            if old + 1 >= self.config.github_token.len() {
+                                wait = true;
+                                Some(0)
+                            } else {
+                                Some(old + 1)
+                            }
+                        })
+                        .unwrap();
+
+                    if wait {
+                        warn!("Tokens wrapped around, sleeping for 1 minute");
+                        sleep(Duration::from_secs(60)).await;
+                    }
+                }
+                err @ Err(_) => return err,
+            }
+
+            // Yield
+            yield_now().await
+        }
+    }
+}
+
+async fn handle_response_json<T: DeserializeOwned>(resp: Response) -> Result<T, Error> {
+    let res = handle_response(resp).await?.json().await?;
+    Ok(res)
+}
+
+/// Converts github responses into the correct error codes (helper for the retry function)
+async fn handle_response(resp: Response) -> Result<Response, Error> {
+    let status = resp.status();
+    if status.is_success() {
+        Ok(resp)
+    } else if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::UNPROCESSABLE_ENTITY
+    {
+        warn!("Rate limit hit");
+        Err(Error::RateLimit(status))
+    } else if let Ok(error) = resp.json().await {
+        let error: GitHubError = error;
+        if error.message.contains("abuse") || error.message.contains("rate limit") {
+            warn!("Rate limit hit ({}): {}", status.as_u16(), error.message);
+            Err(Error::RateLimit(status))
+        } else {
+            warn!("Http Error ({}): {}", status.as_u16(), error.message);
+            Err(Error::HttpError(status))
+        }
+    } else {
+        Err(Error::HttpError(status))
+    }
 }
diff --git a/src/github/mod.rs b/src/github/mod.rs
index e91fd3de3f2..2214cb274fb 100644
--- a/src/github/mod.rs
+++ b/src/github/mod.rs
@@ -1,111 +1,104 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+use api::Github;
+use tokio::{
+    signal::ctrl_c,
+    task::JoinSet,
+    time::{sleep, Instant},
+};
+use tracing::{debug, info, warn};
+
+use crate::{
+    config::Config,
+    data::{Data, Forge},
+};
 
 mod api;
 
-use crate::config::Config;
-use crossbeam_utils::thread::scope;
-use crate::data::{Data, Repo};
-use crate::github::api::GitHubApi;
-use crate::prelude::*;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::time::{Duration, Instant};
-use crate::utils::wrap_thread;
-
-static WANTED_LANG: &str = "Rust";
-
-fn load_thread(api: &GitHubApi, data: &Data, to_load: Vec<String>) -> Fallible<()> {
-    debug!(
-        "collected {} non-fork repositories, loading them",
-        to_load.len()
-    );
-
-    let mut graph_repos = api.load_repositories(&to_load)?;
-    for repo in graph_repos.drain(..).flatten() {
-        let mut found = false;
-        for lang in repo.languages.nodes.iter().filter_map(Option::as_ref) {
-            if lang.name == WANTED_LANG {
-                found = true;
-                break;
-            }
-        }
+#[derive(Debug, Clone)]
+pub struct Scraper {
+    gh: Arc<Github>,
+    data: Data,
+    finished: Arc<AtomicBool>,
+}
+
+impl Scraper {
+    pub fn new(config: Config, data: Data) -> Self {
+        let gh = Github::new(config);
+
+        let finished = Arc::new(AtomicBool::new(false));
+        let f2 = finished.clone();
+
+        tokio::spawn(async move {
+            ctrl_c().await.expect("Failed to install Ctrl+C Handler");
+            warn!("Ctrl+C received, stopping...");
+            f2.store(true, Ordering::SeqCst);
+        });
 
-        if found {
-            let has_cargo_toml = api.file_exists(&repo, "Cargo.toml")?;
-            let has_cargo_lock = api.file_exists(&repo, "Cargo.lock")?;
-
-            data.store_repo(
-                "github",
-                Repo {
-                    id: repo.id,
-                    name: repo.name_with_owner.clone(),
-                    has_cargo_toml,
-                    has_cargo_lock,
-                },
-            )?;
-
-            info!(
-                "found {}: Cargo.toml = {:?}, Cargo.lock = {:?}",
-                repo.name_with_owner, has_cargo_toml, has_cargo_lock,
-            );
+        Self {
+            gh: Arc::new(gh),
+            data,
+            finished,
         }
     }
 
-    // Applease Clippy
-    ::std::mem::drop(to_load);
+    async fn load_repositories(&self, repos: Vec<String>) -> color_eyre::Result<()> {
+        debug!("Loading {} repos", repos.len());
+
+        let mut graph_repos = self.gh.load_repositories(&repos).await?;
+        for repo in graph_repos.drain(..) {
+            if repo
+                .languages
+                .nodes
+                .iter()
+                .filter_map(Option::as_ref)
+                .any(|el| el.name == "Rust")
+            {
+                let mut repo = repo.to_repo(false, false);
+                let files = self.gh.tree(&repo, false).await;
+                match files {
+                    Ok(tree) => {
+                        for node in tree.tree {
+                            if node.path == "Cargo.toml" {
+                                repo.has_cargo_toml = true;
+                            } else if node.path == "Cargo.lock" {
+                                repo.has_cargo_lock = true;
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        warn!("Could not get tree for {}, error: {e:?}", repo.name);
+                    }
+                }
 
-    Ok(())
-}
+                self.data.store_repo(Forge::Github, repo).await;
+            }
+        }
 
-pub fn scrape(data: &Data, config: &Config, should_stop: &AtomicBool) -> Fallible<()> {
-    info!("started scraping for GitHub repositories");
+        Ok(())
+    }
 
-    let gh = api::GitHubApi::new(config);
-    let mut to_load = Vec::with_capacity(100);
+    pub async fn scrape(&self) -> color_eyre::Result<()> {
+        let start = Instant::now();
 
-    let result = scope(|scope| {
-        let mut last_id = data.get_last_id("github")?.unwrap_or(0);
-        let scrape_start = Instant::now();
+        let mut to_load = Vec::with_capacity(100);
 
-        loop {
-            if let Some(timeout) = config.timeout {
-                if scrape_start.elapsed() >= Duration::from_secs(timeout) {
-                    info!("timeout reached, stopping the scraping loop");
-                    break;
-                }
-            }
-
-            // Wait 2 minutes if GitHub is slowing us down
-            if gh.should_slow_down() {
-                warn!("slowing down the scraping (2 minutes pause)");
-                ::std::thread::sleep(Duration::from_secs(120));
-            }
+        let mut last_id = self.data.get_last_id(Forge::Github);
 
-            let start = Instant::now();
+        loop {
+            let start_loop = Instant::now();
+            // TODO check timeout
 
-            debug!("scraping 100 repositories from the REST API");
+            let mut repos = self.gh.scrape_repositories(last_id).await?;
+            let mut js = JoinSet::new();
 
-            // Load all the non-fork repositories in the to_load vector
-            let mut repos = gh.scrape_repositories(last_id)?;
-            let finished = repos.len() < 100 || should_stop.load(Ordering::SeqCst);
-            for repo in repos.drain(..).flatten() {
+            for repo in repos.drain(..) {
                 last_id = repo.id;
                 if repo.fork {
                     continue;
@@ -115,33 +108,35 @@ pub fn scrape(data: &Data, config: &Config, should_stop: &AtomicBool) -> Fallibl
 
                 if to_load.len() == 100 {
                     let to_load_now = to_load.clone();
-                    scope.spawn(|_| wrap_thread(|| load_thread(&gh, data, to_load_now)));
+                    let this = self.clone();
+                    js.spawn(async move { this.load_repositories(to_load_now).await });
                     to_load.clear();
                 }
             }
 
-            data.set_last_id("github", last_id)?;
+            self.data.set_last_id(Forge::Github, last_id).await?;
 
-            if finished {
-                // Ensure all the remaining repositories are loaded
-                if !to_load.is_empty() {
-                    let to_load_now = to_load.clone();
-                    scope.spawn(|_| wrap_thread(|| load_thread(&gh, data, to_load_now)));
+            while let Some(res) = js.join_next().await {
+                let res = res.unwrap();
+                if let Err(e) = res {
+                    warn!("Failed scraping repo: {:?}", e);
                 }
+            }
 
+            if self.finished.load(Ordering::SeqCst) {
+                if !to_load.is_empty() {
+                    self.load_repositories(to_load).await?;
+                }
                 break;
             }
 
-            // Avoid hammering GitHub too much
-            if let Some(sleep) = Duration::from_secs(1).checked_sub(start.elapsed()) {
-                ::std::thread::sleep(sleep);
+            if let Some(time) = Duration::from_millis(250).checked_sub(start_loop.elapsed()) {
+                sleep(time).await;
             }
         }
 
-        Ok(())
-    })
-    .unwrap();
+        info!("Took {} seconds", start.elapsed().as_secs());
 
-    info!("finished scraping for GitHub repositories");
-    result
+        Ok(())
+    }
 }
diff --git a/src/main.rs b/src/main.rs
index c0f45ed852b..b685cfe5eaf 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,129 +1,58 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+use std::{env, path::PathBuf};
 
-extern crate crossbeam_utils;
-extern crate csv;
-extern crate ctrlc;
-extern crate env_logger;
-#[macro_use]
-extern crate failure;
-#[macro_use]
-extern crate log;
-extern crate reqwest;
-extern crate serde;
-#[macro_use]
-extern crate serde_derive;
-#[macro_use]
-extern crate serde_json;
+use config::Config;
+use data::Data;
+use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 mod config;
 mod data;
 mod github;
-mod prelude;
-mod utils;
 
-use config::Config;
-use prelude::*;
-use std::path::PathBuf;
-use std::sync::{
-    atomic::{AtomicBool, Ordering},
-    Arc,
-};
-use std::time::Instant;
+fn get_tokens_from_env() -> Vec<String> {
+    let env_token = env::var("GITHUB_TOKEN").ok();
+    let env_tokens = env::var("GITHUB_TOKENS").ok();
 
-fn app() -> Fallible<()> {
-    // Get the GitHub token from the environment
-    let github_token =
-        std::env::var("GITHUB_TOKEN").context("failed to get the GitHub API token")?;
+    let mut tokens = vec![];
 
-    let timeout = if let Ok(var) = std::env::var("RUST_REPOS_TIMEOUT") {
-        Some(
-            var.parse::<u64>()
-                .context("failed to parse RUST_REPOS_TIMEOUT")?,
-        )
-    } else {
-        None
-    };
-
-    // Parse CLI arguments
-    let args = std::env::args().skip(1).collect::<Vec<String>>();
-    if args.is_empty() {
-        bail!("missing argument: <data_dir>");
-    } else if args.len() > 1 {
-        bail!("too many arguments");
+    if let Some(t) = env_token {
+        tokens.push(t)
     }
 
-    // Ensure the data directory exists
-    let data_dir = PathBuf::from(&args[0]);
-    if !data_dir.is_dir() {
-        debug!(
-            "created missing data directory: {}",
-            data_dir.to_string_lossy()
-        );
-        std::fs::create_dir_all(&data_dir)?;
+    if let Some(ts) = env_tokens {
+        let ts = ts.split(',');
+        for t in ts {
+            tokens.push(t.to_string())
+        }
     }
 
-    let config = Config {
-        github_token,
-        data_dir,
-        timeout,
-    };
-
-    let data = data::Data::new(&config);
+    tokens
+}
 
-    let should_stop = Arc::new(AtomicBool::new(false));
-    let stop = should_stop.clone();
-    ctrlc::set_handler(move || {
-        info!("received Ctrl+C, terminating...");
-        stop.store(true, Ordering::SeqCst);
-    })?;
+#[tokio::main]
+async fn main() -> color_eyre::Result<()> {
+    dotenvy::dotenv().ok();
+    tracing_subscriber::registry()
+        .with(fmt::layer())
+        .with(EnvFilter::from_default_env())
+        .init();
 
-    github::scrape(&data, &config, &should_stop)?;
+    let github_token = get_tokens_from_env();
 
-    Ok(())
-}
+    let timeout = env::var("RUST_REPOS_TIMEOUT")
+        .ok()
+        .and_then(|el| el.parse::<u64>().ok());
 
-fn main() {
-    // Initialize logging
-    // This doesn't use from_default_env() because it doesn't allow to override filter_module()
-    // with the RUST_LOG environment variable
-    let mut logger = env_logger::Builder::new();
-    logger.filter_module("rust_repos", log::LevelFilter::Info);
-    if let Ok(content) = std::env::var("RUST_LOG") {
-        logger.parse_filters(&content);
-    }
-    logger.init();
+    let data_dir = PathBuf::from("./data_new");
 
-    let start = Instant::now();
+    let config = Config {
+        github_token,
+        data_dir,
+        timeout,
+    };
 
-    let result = app();
-    if let Err(ref err) = &result {
-        utils::log_error(err);
-    }
+    let data = Data::new(&config)?;
 
-    info!(
-        "execution completed in {} seconds",
-        start.elapsed().as_secs()
-    );
+    let scraper = github::Scraper::new(config, data);
 
-    if result.is_err() {
-        std::process::exit(1);
-    }
+    scraper.scrape().await
 }
diff --git a/src/prelude.rs b/src/prelude.rs
deleted file mode 100644
index 57bc2552e2e..00000000000
--- a/src/prelude.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-pub use failure::{err_msg, Error, Fail, Fallible, ResultExt};
diff --git a/src/utils.rs b/src/utils.rs
deleted file mode 100644
index c7d40b34b49..00000000000
--- a/src/utils.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright (c) 2018 Pietro Albini <pietro@pietroalbini.org>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of
-// this software and associated documentation files (the "Software"), to deal in
-// the Software without restriction, including without limitation the rights to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to do
-// so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-use crate::prelude::*;
-
-pub fn log_error(err: &Error) {
-    error!("{}", err);
-    for cause in err.iter_causes() {
-        error!("  caused by: {}", cause);
-    }
-}
-
-pub fn wrap_thread<F: FnOnce() -> Fallible<()>>(f: F) {
-    if let Err(err) = f() {
-        log_error(&err);
-    }
-}

From a2b2bc7be9a26b887116fe86b8e0a54ca71a85d5 Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Mon, 21 Oct 2024 11:19:45 +0200
Subject: [PATCH 4/8] made Data into an Arc<InnerData>

---
 src/data.rs       | 56 ++++++++++++++++++++++++-----------------------
 src/github/api.rs |  2 +-
 src/github/mod.rs |  2 +-
 3 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/src/data.rs b/src/data.rs
index 16cbf18c761..8b989d8fc91 100644
--- a/src/data.rs
+++ b/src/data.rs
@@ -2,7 +2,7 @@ use enum_map::{Enum, EnumMap};
 use serde_derive::{Deserialize, Serialize};
 use tokio::sync::Mutex;
 use tokio::task::{spawn_blocking, JoinSet};
-use tracing::{debug, info, trace};
+use tracing::debug;
 
 use crate::config::Config;
 use std::collections::BTreeMap;
@@ -31,64 +31,66 @@ pub struct Repo {
     pub has_cargo_lock: bool,
 }
 
-#[derive(Debug, Clone)]
-pub struct Data {
+#[derive(Debug)]
+pub struct InnerData {
     data_dir: PathBuf,
 
-    state_lock: Arc<Mutex<()>>,
+    state_lock: Mutex<()>,
 
-    state_cache: Arc<State>,
+    state_cache: State,
 
-    repos_state: Arc<Mutex<EnumMap<Forge, BTreeMap<String, Repo>>>>,
+    repos_state: Mutex<EnumMap<Forge, BTreeMap<String, Repo>>>,
 }
 
+#[derive(Debug, Clone)]
+pub struct Data(Arc<InnerData>);
+
 impl Data {
     pub fn new(config: &Config) -> color_eyre::Result<Self> {
         fs::create_dir_all(&config.data_dir)?;
 
-        let mut data = Data {
-            data_dir: config.data_dir.clone(),
-
-            state_lock: Arc::new(Mutex::new(())),
-            state_cache: Arc::new(State::default()),
-            repos_state: Arc::new(Mutex::new(EnumMap::default())),
+        let state_path = config.data_dir.join("state.json");
+        let state_cache = if state_path.exists() {
+            serde_json::from_slice(&fs::read(&state_path)?)?
+        } else {
+            State::default()
         };
 
-        let state_path = data.state_path();
-        if state_path.exists() {
-            let state_cache: State = serde_json::from_slice(&fs::read(&state_path)?)?;
+        let data = Self(Arc::new(InnerData {
+            data_dir: config.data_dir.clone(),
 
-            data.state_cache = Arc::new(state_cache)
-        }
+            state_lock: Mutex::new(()),
+            state_cache,
+            repos_state: Mutex::new(EnumMap::default()),
+        }));
 
         Ok(data)
     }
 
     pub fn state_path(&self) -> PathBuf {
-        self.data_dir.join("state.json")
+        self.0.data_dir.join("state.json")
     }
 
     pub fn csv_path(&self, forge: Forge) -> PathBuf {
         match forge {
-            Forge::Github => self.data_dir.join("github.csv"),
+            Forge::Github => self.0.data_dir.join("github.csv"),
         }
     }
 
     pub fn get_last_id(&self, forge: Forge) -> usize {
-        self.state_cache.0[forge].load(std::sync::atomic::Ordering::SeqCst)
+        self.0.state_cache.0[forge].load(std::sync::atomic::Ordering::SeqCst)
     }
 
     /// Store the state cache to disk, i.e. last fetched ids
     async fn store_state_cache(&self) -> color_eyre::Result<()> {
-        let state = self.state_cache.clone();
-        let lock = self.state_lock.clone();
+        let this = self.clone();
         let state_path = self.state_path();
         spawn_blocking(move || -> color_eyre::Result<()> {
-            let guard = lock.blocking_lock();
+            let guard = this.0.state_lock.blocking_lock();
 
             let file = File::create(state_path)?;
             let mut file = BufWriter::new(file);
-            serde_json::to_writer_pretty(&mut file, state.as_ref())?;
+            serde_json::to_writer_pretty(&mut file, &this.0.state_cache)?;
             file.write_all(b"\n")?;
 
             drop(guard);
@@ -102,7 +104,7 @@ impl Data {
     /// Stores the repos found to disk in a CSV
     async fn store_csv(&self) -> color_eyre::Result<()> {
         debug!("storing csv file");
-        let mut repos = self.repos_state.lock().await;
+        let mut repos = self.0.repos_state.lock().await;
 
         let mut js = JoinSet::new();
 
@@ -139,7 +141,7 @@ impl Data {
     }
 
     pub async fn set_last_id(&self, forge: Forge, n: usize) -> color_eyre::Result<()> {
-        self.state_cache.0[forge].store(n, std::sync::atomic::Ordering::SeqCst);
+        self.0.state_cache.0[forge].store(n, std::sync::atomic::Ordering::SeqCst);
 
         self.store_csv().await?;
         self.store_state_cache().await?;
@@ -148,7 +150,7 @@ impl Data {
     }
 
     pub async fn store_repo(&self, forge: Forge, repo: Repo) {
-        let mut repos_state = self.repos_state.lock().await;
+        let mut repos_state = self.0.repos_state.lock().await;
         repos_state[forge].insert(repo.name.clone(), repo);
     }
 }
diff --git a/src/github/api.rs b/src/github/api.rs
index 09e87969f8a..ab092de3d7c 100644
--- a/src/github/api.rs
+++ b/src/github/api.rs
@@ -78,7 +78,7 @@ pub struct GraphRepository {
 }
 
 impl GraphRepository {
-    pub fn to_repo(self, has_cargo_toml: bool, has_cargo_lock: bool) -> Repo {
+    pub fn into_repo(self, has_cargo_toml: bool, has_cargo_lock: bool) -> Repo {
         Repo {
             id: self.id,
             name: self.name_with_owner,
diff --git a/src/github/mod.rs b/src/github/mod.rs
index 2214cb274fb..91ea33d7429 100644
--- a/src/github/mod.rs
+++ b/src/github/mod.rs
@@ -60,7 +60,7 @@ impl Scraper {
                 .filter_map(Option::as_ref)
                 .any(|el| el.name == "Rust")
             {
-                let mut repo = repo.to_repo(false, false);
+                let mut repo = repo.into_repo(false, false);
                 let files = self.gh.tree(&repo, false).await;
                 match files {
                     Ok(tree) => {

From e036fcf09b451225f22774e2cb7d1b03cd532a9b Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Mon, 21 Oct 2024 12:29:03 +0200
Subject: [PATCH 5/8] cleanup

---
 .cargo/config.toml |   2 +
 .gitignore         |   1 +
 Cargo.lock         | 476 ++++++++++++++++++++++++++++++++++++++++++++-
 Cargo.toml         |   2 +
 src/github/api.rs  |  51 +++--
 src/github/mod.rs  |  45 +++--
 src/main.rs        |   4 +-
 7 files changed, 536 insertions(+), 45 deletions(-)
 create mode 100644 .cargo/config.toml

diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 00000000000..16f1e73ac06
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable"]
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 37c8717f9a0..6bca7c191b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /target
 **/*.rs.bk
 .env
+/data_new
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 88242eeef3c..9a0c3cf29e1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -26,12 +32,104 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.90"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95"
+
+[[package]]
+name = "async-stream"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "axum"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.66"
@@ -42,11 +140,17 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide",
+ "miniz_oxide 0.5.4",
  "object",
  "rustc-demangle",
 ]
 
+[[package]]
+name = "base64"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -119,6 +223,69 @@ dependencies = [
  "tracing-error",
 ]
 
+[[package]]
+name = "console-api"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86ed14aa9c9f927213c6e4f3ef75faaad3406134efe84ba2cb7983431d5f0931"
+dependencies = [
+ "futures-core",
+ "prost",
+ "prost-types",
+ "tonic",
+ "tracing-core",
+]
+
+[[package]]
+name = "console-subscriber"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e3a111a37f3333946ebf9da370ba5c5577b18eb342ec683eb488dd21980302"
+dependencies = [
+ "console-api",
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "futures-task",
+ "hdrhistogram",
+ "humantime",
+ "hyper-util",
+ "prost",
+ "prost-types",
+ "serde",
+ "serde_json",
+ "thread_local",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+
 [[package]]
 name = "csv"
 version = "1.3.0"
@@ -146,6 +313,12 @@ version = "0.15.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
 
+[[package]]
+name = "either"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+
 [[package]]
 name = "enum-map"
 version = "2.7.3"
@@ -167,6 +340,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "eyre"
 version = "0.6.12"
@@ -177,6 +356,16 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide 0.8.0",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -259,6 +448,50 @@ version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
 
+[[package]]
+name = "h2"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap 2.6.0",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hashbrown"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
+
+[[package]]
+name = "hdrhistogram"
+version = "7.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
+dependencies = [
+ "base64 0.21.7",
+ "byteorder",
+ "flate2",
+ "nom",
+ "num-traits",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.9"
@@ -305,6 +538,18 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
 [[package]]
 name = "hyper"
 version = "1.4.1"
@@ -314,9 +559,11 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
+ "h2",
  "http",
  "http-body",
  "httparse",
+ "httpdate",
  "itoa",
  "pin-project-lite",
  "smallvec",
@@ -342,6 +589,19 @@ dependencies = [
  "webpki-roots",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793"
+dependencies = [
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
+]
+
 [[package]]
 name = "hyper-util"
 version = "0.1.9"
@@ -377,17 +637,46 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
 
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.15.0",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
-version = "1.0.4"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
 [[package]]
 name = "js-sys"
@@ -435,6 +724,12 @@ dependencies = [
  "regex-automata 0.1.10",
 ]
 
+[[package]]
+name = "matchit"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -447,6 +742,12 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.4"
@@ -456,6 +757,15 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+dependencies = [
+ "adler2",
+]
+
 [[package]]
 name = "mio"
 version = "1.0.2"
@@ -468,6 +778,16 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -478,6 +798,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "object"
 version = "0.29.0"
@@ -534,6 +863,26 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
+[[package]]
+name = "pin-project"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.14"
@@ -564,6 +913,38 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670"
+dependencies = [
+ "prost",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.5"
@@ -710,7 +1091,7 @@ version = "0.12.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
 dependencies = [
- "base64",
+ "base64 0.22.1",
  "bytes",
  "futures-channel",
  "futures-core",
@@ -767,6 +1148,7 @@ name = "rust-repos"
 version = "0.2.0"
 dependencies = [
  "color-eyre",
+ "console-subscriber",
  "csv",
  "dotenvy",
  "enum-map",
@@ -832,6 +1214,12 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "rustversion"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
+
 [[package]]
 name = "ryu"
 version = "1.0.11"
@@ -1055,6 +1443,86 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tonic"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum",
+ "base64 0.22.1",
+ "bytes",
+ "h2",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-timeout",
+ "hyper-util",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "socket2",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap 1.9.3",
+ "pin-project",
+ "pin-project-lite",
+ "rand",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
+
 [[package]]
 name = "tower-service"
 version = "0.3.2"
diff --git a/Cargo.toml b/Cargo.toml
index 84e48c8e65e..50c7f40ce73 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,3 +25,5 @@ color-eyre = "0.6.3"
 tracing = "0.1.40"
 enum-map = { version = "2.7.3", features = ["serde"] }
 thiserror = "1"
+
+console-subscriber = "0.4"
diff --git a/src/github/api.rs b/src/github/api.rs
index ab092de3d7c..6ac6b5e95bb 100644
--- a/src/github/api.rs
+++ b/src/github/api.rs
@@ -12,7 +12,7 @@ use serde_derive::Deserialize;
 use serde_json::json;
 use thiserror::Error;
 use tokio::{task::yield_now, time::sleep};
-use tracing::{debug, error, warn};
+use tracing::{error, trace, warn};
 
 use crate::{config::Config, data::Repo};
 
@@ -25,11 +25,12 @@ pub struct Github {
     current_token_index: AtomicUsize,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize)]
 pub struct GitHubError {
     message: String,
-    #[serde(rename = "type")]
-    type_: Option<String>,
+
+    #[allow(unused)]
+    r#type: Option<String>,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -45,6 +46,7 @@ pub struct GithubTree {
 #[derive(Debug, Deserialize)]
 pub struct RestRepository {
     pub id: usize,
+    #[allow(unused, reason = "Useful for debugging, if something does go wrong")]
     pub full_name: String,
     pub node_id: String,
     pub fork: bool,
@@ -98,11 +100,6 @@ pub struct GraphLanguage {
     pub name: String,
 }
 
-#[derive(Debug, Deserialize)]
-pub struct GraphRef {
-    pub name: String,
-}
-
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("reqwest error occurred {0:?}")]
@@ -110,7 +107,7 @@ pub enum Error {
     #[error("rate limit hit {0}")]
     RateLimit(StatusCode),
     #[error("other http error: {0}")]
-    HttpError(StatusCode),
+    HttpStatus(StatusCode),
 
     #[error("Response did not contain requested data")]
     EmptyData,
@@ -118,6 +115,9 @@ pub enum Error {
     Io(#[from] std::io::Error),
 }
 
+/// 100 is the max results per page of the GH API
+pub(crate) const N: usize = 100;
+
 const GRAPHQL_QUERY_REPOSITORIES: &str = "
 query($ids: [ID!]!) {
     nodes(ids: $ids) {
@@ -153,12 +153,12 @@ impl Github {
     }
 
     fn build_request(&self, method: Method, url: &str) -> RequestBuilder {
-        let url = if !url.starts_with("https://") {
-            Cow::from(format!("https://api.github.com/{}", url))
-        } else {
+        let url = if url.starts_with("https://") {
             Cow::from(url)
+        } else {
+            Cow::from(format!("https://api.github.com/{url}"))
         };
-        debug!("Sending request to {url}");
+        trace!("Sending request to {url}");
         self.client
             .request(method, url.as_ref())
             .header(header::AUTHORIZATION, format!("token {}", self.get_token()))
@@ -182,6 +182,10 @@ impl Github {
 
         let data: GraphResponse<T> = handle_response_json(resp).await?;
 
+        if let Some(errs) = data.errors {
+            warn!("GraphQL Errors: {:?}, \n {:#?}", data.message, errs);
+        }
+
         data.data.ok_or_else(|| Error::EmptyData)
     }
 
@@ -213,7 +217,7 @@ impl Github {
     pub async fn tree(&self, repo: &Repo, recursive: bool) -> Result<GithubTree, Error> {
         let mut url = format!("repos/{}/git/trees/HEAD", repo.name);
         if recursive {
-            url = format!("{url}?recursive=1")
+            url = format!("{url}?recursive=1");
         }
 
         self.retry(|| async {
@@ -230,7 +234,10 @@ impl Github {
         let output: Vec<RestRepository> = self
             .retry(|| async {
                 let resp = self
-                    .build_request(Method::GET, &format!("repositories?since={}", since))
+                    .build_request(
+                        Method::GET,
+                        &format!("repositories?since={since}&per_page{N}"),
+                    )
                     .send()
                     .await?;
 
@@ -238,6 +245,10 @@ impl Github {
             })
             .await?;
 
+        if output.len() != N {
+            warn!("Github API returned {} instead of {N} repos", output.len());
+        }
+
         Ok(output)
     }
 
@@ -265,7 +276,7 @@ impl Github {
                         return Err(Error::Reqwest(reqwest_error));
                     }
                 }
-                Err(err @ Error::HttpError(_)) => return Err(err),
+                Err(err @ Error::HttpStatus(_)) => return Err(err),
                 Err(Error::RateLimit(_)) => {
                     let mut wait = false;
                     self.current_token_index
@@ -288,7 +299,7 @@ impl Github {
             }
 
             // Yield
-            yield_now().await
+            yield_now().await;
         }
     }
 }
@@ -314,9 +325,9 @@ async fn handle_response(resp: Response) -> Result<Response, Error> {
             Err(Error::RateLimit(status))
         } else {
             warn!("Http Error ({}): {}", status.as_u16(), error.message);
-            Err(Error::HttpError(status))
+            Err(Error::HttpStatus(status))
         }
     } else {
-        Err(Error::HttpError(status))
+        Err(Error::HttpStatus(status))
     }
 }
diff --git a/src/github/mod.rs b/src/github/mod.rs
index 91ea33d7429..16a67e05f53 100644
--- a/src/github/mod.rs
+++ b/src/github/mod.rs
@@ -49,9 +49,8 @@ impl Scraper {
     }
 
     async fn load_repositories(&self, repos: Vec<String>) -> color_eyre::Result<()> {
-        debug!("Loading {} repos", repos.len());
-
         let mut graph_repos = self.gh.load_repositories(&repos).await?;
+        let mut js = JoinSet::new();
         for repo in graph_repos.drain(..) {
             if repo
                 .languages
@@ -60,34 +59,39 @@ impl Scraper {
                 .filter_map(Option::as_ref)
                 .any(|el| el.name == "Rust")
             {
-                let mut repo = repo.into_repo(false, false);
-                let files = self.gh.tree(&repo, false).await;
-                match files {
-                    Ok(tree) => {
-                        for node in tree.tree {
-                            if node.path == "Cargo.toml" {
-                                repo.has_cargo_toml = true;
-                            } else if node.path == "Cargo.lock" {
-                                repo.has_cargo_lock = true;
+                let this = self.clone();
+                js.spawn(async move {
+                    let mut repo = repo.into_repo(false, false);
+                    let files = this.gh.tree(&repo, false).await;
+                    match files {
+                        Ok(tree) => {
+                            for node in tree.tree {
+                                if node.path == "Cargo.toml" {
+                                    repo.has_cargo_toml = true;
+                                } else if node.path == "Cargo.lock" {
+                                    repo.has_cargo_lock = true;
+                                }
                             }
                         }
+                        Err(e) => {
+                            warn!("Could not get tree for {}, error: {e:?}", repo.name);
+                        }
                     }
-                    Err(e) => {
-                        warn!("Could not get tree for {}, error: {e:?}", repo.name);
-                    }
-                }
 
-                self.data.store_repo(Forge::Github, repo).await;
+                    this.data.store_repo(Forge::Github, repo).await;
+                });
             }
         }
 
+        js.join_all().await;
+
         Ok(())
     }
 
     pub async fn scrape(&self) -> color_eyre::Result<()> {
         let start = Instant::now();
 
-        let mut to_load = Vec::with_capacity(100);
+        let mut to_load = Vec::with_capacity(api::N);
 
         let mut last_id = self.data.get_last_id(Forge::Github);
 
@@ -106,7 +110,7 @@ impl Scraper {
 
                 to_load.push(repo.node_id);
 
-                if to_load.len() == 100 {
+                if to_load.len() == api::N {
                     let to_load_now = to_load.clone();
                     let this = self.clone();
                     js.spawn(async move { this.load_repositories(to_load_now).await });
@@ -116,8 +120,7 @@ impl Scraper {
 
             self.data.set_last_id(Forge::Github, last_id).await?;
 
-            while let Some(res) = js.join_next().await {
-                let res = res.unwrap();
+            for res in js.join_all().await {
                 if let Err(e) = res {
                     warn!("Failed scraping repo: {:?}", e);
                 }
@@ -130,6 +133,8 @@ impl Scraper {
                 break;
             }
 
+            debug!("Loaded 100 repos in {}ms", start_loop.elapsed().as_millis());
+
             if let Some(time) = Duration::from_millis(250).checked_sub(start_loop.elapsed()) {
                 sleep(time).await;
             }
diff --git a/src/main.rs b/src/main.rs
index b685cfe5eaf..cd32011e378 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -21,7 +21,7 @@ fn get_tokens_from_env() -> Vec<String> {
     if let Some(ts) = env_tokens {
         let ts = ts.split(',');
         for t in ts {
-            tokens.push(t.to_string())
+            tokens.push(t.to_string());
         }
     }
 
@@ -36,6 +36,8 @@ async fn main() -> color_eyre::Result<()> {
         .with(EnvFilter::from_default_env())
         .init();
 
+    // console_subscriber::init();
+
     let github_token = get_tokens_from_env();
 
     let timeout = env::var("RUST_REPOS_TIMEOUT")

From f83b35e9e81d192f59254737ee44bad0d748a98f Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Mon, 21 Oct 2024 12:35:13 +0200
Subject: [PATCH 6/8] add timeout

---
 src/github/mod.rs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/github/mod.rs b/src/github/mod.rs
index 16a67e05f53..c8609105b90 100644
--- a/src/github/mod.rs
+++ b/src/github/mod.rs
@@ -26,10 +26,12 @@ pub struct Scraper {
     gh: Arc<Github>,
     data: Data,
     finished: Arc<AtomicBool>,
+    timeout: Option<u64>,
 }
 
 impl Scraper {
     pub fn new(config: Config, data: Data) -> Self {
+        let timeout = config.timeout;
         let gh = Github::new(config);
 
         let finished = Arc::new(AtomicBool::new(false));
@@ -45,6 +47,7 @@ impl Scraper {
             gh: Arc::new(gh),
             data,
             finished,
+            timeout,
         }
     }
 
@@ -97,7 +100,12 @@ impl Scraper {
 
         loop {
             let start_loop = Instant::now();
-            // TODO check timeout
+            if let Some(timeout) = self.timeout {
+                if start.elapsed() >= Duration::from_secs(timeout) {
+                    info!("Timeout reached, stopped scraping");
+                    break;
+                }
+            }
 
             let mut repos = self.gh.scrape_repositories(last_id).await?;
             let mut js = JoinSet::new();

From 9e1f87dd85e9cc13bd363cd7f223222b2c836425 Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Fri, 25 Oct 2024 09:48:48 +0200
Subject: [PATCH 7/8] small cleanup

---
 .cargo/config.toml | 2 --
 src/config.rs      | 1 +
 src/data.rs        | 1 +
 src/main.rs        | 4 +---
 4 files changed, 3 insertions(+), 5 deletions(-)
 delete mode 100644 .cargo/config.toml

diff --git a/.cargo/config.toml b/.cargo/config.toml
deleted file mode 100644
index 16f1e73ac06..00000000000
--- a/.cargo/config.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-[build]
-rustflags = ["--cfg", "tokio_unstable"]
\ No newline at end of file
diff --git a/src/config.rs b/src/config.rs
index ee65bc977b2..e579e5e15ac 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -2,6 +2,7 @@ use std::path::PathBuf;
 
 #[derive(Debug, Clone)]
 pub struct Config {
+    // GitHub rate-limits are per-token, using more than one can speed up the scraper
     pub github_token: Vec<String>,
     pub data_dir: PathBuf,
     pub timeout: Option<u64>,
diff --git a/src/data.rs b/src/data.rs
index 8b989d8fc91..9eaab56aba8 100644
--- a/src/data.rs
+++ b/src/data.rs
@@ -17,6 +17,7 @@ use std::{
 
 #[derive(Debug, Enum, Serialize, Deserialize, Copy, Clone)]
 pub enum Forge {
+    #[serde(rename = "github")]
     Github,
 }
 
diff --git a/src/main.rs b/src/main.rs
index cd32011e378..2d262c607af 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,7 +15,7 @@ fn get_tokens_from_env() -> Vec<String> {
     let mut tokens = vec![];
 
     if let Some(t) = env_token {
-        tokens.push(t)
+        tokens.push(t);
     }
 
     if let Some(ts) = env_tokens {
@@ -36,8 +36,6 @@ async fn main() -> color_eyre::Result<()> {
         .with(EnvFilter::from_default_env())
         .init();
 
-    // console_subscriber::init();
-
     let github_token = get_tokens_from_env();
 
     let timeout = env::var("RUST_REPOS_TIMEOUT")

From ce7478070d93181149ab3fd19530247aebbeb7ab Mon Sep 17 00:00:00 2001
From: Vivian <vivian@0x76.dev>
Date: Fri, 25 Oct 2024 10:44:36 +0200
Subject: [PATCH 8/8] ignore NOT_FOUND errors

---
 src/github/api.rs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/github/api.rs b/src/github/api.rs
index 6ac6b5e95bb..41b3dd70eac 100644
--- a/src/github/api.rs
+++ b/src/github/api.rs
@@ -46,7 +46,8 @@ pub struct GithubTree {
 #[derive(Debug, Deserialize)]
 pub struct RestRepository {
     pub id: usize,
-    #[allow(unused, reason = "Useful for debugging, if something does go wrong")]
+    // Useful for debugging, if something does go wrong
+    #[allow(unused)]
     pub full_name: String,
     pub node_id: String,
     pub fork: bool,
@@ -56,6 +57,7 @@ pub struct RestRepository {
 struct GraphResponse<T> {
     data: Option<T>,
     errors: Option<Vec<GitHubError>>,
+    #[allow(unused)]
     message: Option<String>,
 }
 
@@ -183,7 +185,14 @@ impl Github {
         let data: GraphResponse<T> = handle_response_json(resp).await?;
 
         if let Some(errs) = data.errors {
-            warn!("GraphQL Errors: {:?}, \n {:#?}", data.message, errs);
+            let errs: Vec<_> = errs
+                .into_iter()
+                // Some repos on GitHub are just marked as NOT_FOUND, does not seem like our fault
+                .filter(|el| el.r#type.as_deref() != Some("NOT_FOUND"))
+                .collect();
+            if !errs.is_empty() {
+                warn!("GraphQL Errors: \n {:#?}", errs);
+            }
         }
 
         data.data.ok_or_else(|| Error::EmptyData)