diff --git a/.gitignore b/.gitignore index 53eaa21960d..6bca7c191b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /target **/*.rs.bk +.env +/data_new \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index daf2abd983f..9a0c3cf29e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,32 +17,119 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" -version = "0.7.19" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] -name = "atty" -version = "0.2.14" +name = "anyhow" +version = "1.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" + +[[package]] +name = "async-stream" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ - "hermit-abi", - "libc", - "winapi", + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.66" @@ -53,34 +140,28 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.5.4", "object", "rustc-demangle", ] [[package]] name = "base64" -version = "0.13.0" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] -name = "bitflags" -version = "1.3.2" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "bstr" -version = "0.2.17" +name = "bitflags" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bumpalo" @@ -88,17 +169,26 @@ version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" -version = "1.2.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.0.73" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -107,141 +197,186 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "core-foundation" -version = "0.9.3" +name = "color-eyre" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" dependencies = [ - "core-foundation-sys", - "libc", + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", ] [[package]] -name = "core-foundation-sys" -version = "0.8.3" +name = "color-spantrace" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] [[package]] -name = "crossbeam-utils" -version = "0.8.12" +name = "console-api" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +checksum = "86ed14aa9c9f927213c6e4f3ef75faaad3406134efe84ba2cb7983431d5f0931" dependencies = [ - "cfg-if", + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", ] [[package]] -name = "csv" -version = "1.1.6" +name = "console-subscriber" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +checksum = "e2e3a111a37f3333946ebf9da370ba5c5577b18eb342ec683eb488dd21980302" dependencies = [ - "bstr", - "csv-core", - "itoa 0.4.8", - "ryu", + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", ] [[package]] -name = "csv-core" -version = "0.1.10" +name = "crc32fast" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "memchr", + "cfg-if", ] [[package]] -name = "ctrlc" -version = "3.2.3" +name = "crossbeam-channel" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ - "nix", - "winapi", + "crossbeam-utils", ] [[package]] -name = "encoding_rs" -version = "0.8.31" +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "csv" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ - "cfg-if", + "csv-core", + "itoa", + "ryu", + "serde", ] [[package]] -name = "env_logger" -version = "0.9.1" +name = "csv-core" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", + "memchr", ] [[package]] -name = "failure" -version = "0.1.8" +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "enum-map" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" dependencies = [ - "backtrace", - "failure_derive", + "enum-map-derive", + "serde", ] [[package]] -name = "failure_derive" -version = "0.1.8" +name = "enum-map-derive" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", "syn", - "synstructure", ] [[package]] -name = "fastrand" -version = "1.8.0" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" -dependencies = [ - "instant", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "fnv" -version = "1.0.7" +name = "eyre" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] [[package]] -name = "foreign-types" -version = "0.3.2" +name = "flate2" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ - "foreign-types-shared", + "crc32fast", + "miniz_oxide 0.8.0", ] [[package]] -name = "foreign-types-shared" -version = "0.1.1" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -253,40 +388,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" dependencies = [ "futures-core", + "futures-sink", ] [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-io", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -294,6 +431,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.26.2" @@ -302,17 +450,17 @@ checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" [[package]] name = "h2" -version = "0.3.14" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http", - "indexmap", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -326,33 +474,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + +[[package]] +name = "hdrhistogram" +version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "libc", + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", ] +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "http" -version = "0.2.8" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa", ] [[package]] name = "http-body" -version = "0.4.5" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", "pin-project-lite", ] @@ -364,9 +540,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -376,68 +552,109 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.20" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", "http", "http-body", "httparse", "httpdate", - "itoa 1.0.4", + "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", "hyper", - "native-tls", + "pin-project-lite", + "socket2", "tokio", - "tokio-native-tls", + "tower-service", + "tracing", ] [[package]] name = "idna" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] -name = "instant" -version = "0.1.12" +name = "indexmap" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ - "cfg-if", + "equivalent", + "hashbrown 0.15.0", ] [[package]] @@ -447,16 +664,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] -name = "itoa" -version = "0.4.8" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" @@ -469,25 +689,47 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.135" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] [[package]] name = "log" -version = "0.4.17" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "matchers" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "cfg-if", + "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.5.0" @@ -500,6 +742,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.5.4" @@ -509,56 +757,54 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" -version = "0.8.4" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi", "libc", - "log", "wasi", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] -name = "native-tls" -version = "0.2.10" +name = "nom" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", + "memchr", + "minimal-lexical", ] [[package]] -name = "nix" -version = "0.25.0" +name = "nu-ansi-term" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ - "autocfg", - "bitflags", - "cfg-if", - "libc", + "overload", + "winapi", ] [[package]] -name = "num_cpus" -version = "1.13.1" +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "hermit-abi", - "libc", + "autocfg", ] [[package]] @@ -572,115 +818,239 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.15.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] -name = "openssl" -version = "0.10.42" +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "parking_lot" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "bitflags", "cfg-if", - "foreign-types", "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "redox_syscall", + "smallvec", + "windows-targets", ] [[package]] -name = "openssl-macros" +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +dependencies = [ + "anyhow", + "itertools", "proc-macro2", "quote", "syn", ] [[package]] -name = "openssl-probe" -version = "0.1.5" +name = "prost-types" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +dependencies = [ + "prost", +] [[package]] -name = "openssl-sys" -version = "0.9.76" +name = "quinn" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", ] [[package]] -name = "percent-encoding" -version = "2.2.0" +name = "quinn-proto" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] [[package]] -name = "pin-project-lite" -version = "0.2.9" +name = "quinn-udp" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] [[package]] -name = "pin-utils" -version = "0.1.0" +name = "quote" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] [[package]] -name = "pkg-config" -version = "0.3.25" +name = "rand" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] [[package]] -name = "proc-macro2" -version = "1.0.47" +name = "rand_chacha" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ - "unicode-ident", + "ppv-lite86", + "rand_core", ] [[package]] -name = "quote" -version = "1.0.21" +name = "rand_core" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "proc-macro2", + "getrandom", ] [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.6.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] @@ -688,73 +1058,108 @@ name = "regex-automata" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", +] [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "regex-syntax" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.12" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ - "base64", + "base64 0.22.1", "bytes", - "encoding_rs", + "futures-channel", "futures-core", "futures-util", - "h2", "http", "http-body", + "http-body-util", "hyper", - "hyper-tls", + "hyper-rustls", + "hyper-util", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "webpki-roots", + "windows-registry", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", ] [[package]] name = "rust-repos" -version = "0.1.0" +version = "0.2.0" dependencies = [ - "crossbeam-utils", + "color-eyre", + "console-subscriber", "csv", - "ctrlc", - "env_logger", - "failure", - "log", + "dotenvy", + "enum-map", "reqwest", "serde", "serde_derive", "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -764,55 +1169,83 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] -name = "ryu" -version = "1.0.11" +name = "rustc-hash" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] -name = "schannel" -version = "0.1.20" +name = "rustls" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ - "lazy_static", - "windows-sys", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", ] [[package]] -name = "security-framework" -version = "2.7.0" +name = "rustls-pemfile" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", + "rustls-pki-types", ] [[package]] -name = "security-framework-sys" -version = "2.6.1" +name = "rustls-pki-types" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ - "core-foundation-sys", - "libc", + "ring", + "rustls-pki-types", + "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" -version = "1.0.147" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -821,11 +1254,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.87" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "itoa 1.0.4", + "itoa", + "memchr", "ryu", "serde", ] @@ -837,11 +1271,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa", "ryu", "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.7" @@ -851,21 +1309,39 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "socket2" -version = "0.4.7" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "winapi", + "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" -version = "1.0.102" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -873,38 +1349,42 @@ dependencies = [ ] [[package]] -name = "synstructure" -version = "0.12.6" +name = "sync_wrapper" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "futures-core", ] [[package]] -name = "tempfile" -version = "3.3.0" +name = "thiserror" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ - "cfg-if", - "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", + "thiserror-impl", ] [[package]] -name = "termcolor" -version = "1.1.3" +name = "thiserror-impl" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "winapi-util", + "cfg-if", + "once_cell", ] [[package]] @@ -924,45 +1404,125 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.2" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", - "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", - "winapi", + "tokio-macros", + "tracing", + "windows-sys 0.52.0", ] [[package]] -name = "tokio-native-tls" -version = "0.3.0" +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ - "native-tls", + "futures-core", + "pin-project-lite", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", "tracing", ] +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.2" @@ -971,22 +1531,73 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", + "nu-ansi-term", "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -997,9 +1608,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -1017,16 +1628,16 @@ dependencies = [ ] [[package]] -name = "unicode-xid" -version = "0.2.4" +name = "untrusted" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.3.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -1034,10 +1645,10 @@ dependencies = [ ] [[package]] -name = "vcpkg" -version = "0.2.15" +name = "valuable" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "want" @@ -1057,19 +1668,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -1094,9 +1706,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1104,9 +1716,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -1117,9 +1729,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" @@ -1131,6 +1743,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1148,68 +1769,146 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] -name = "winapi-util" -version = "0.1.5" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "winapi", + "windows-result", + "windows-strings", + "windows-targets", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "windows-result" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets", +] [[package]] name = "windows-sys" -version = "0.36.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", + "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", + "windows_x86_64_gnullvm", "windows_x86_64_msvc", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" -version = "0.36.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.36.1" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.36.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winreg" -version = "0.10.1" +name = "zerocopy" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "winapi", + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml index e2d6fe51b5a..50c7f40ce73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,29 @@ [package] name = "rust-repos" -version = "0.1.0" -authors = ["Pietro Albini "] +edition = "2021" +version = "0.2.0" +authors = [ + "Pietro Albini ", + "Vivian Roest ", +] [dependencies] -failure = "0.1.5" -reqwest = { version = "0.11.12", features = ["blocking", "json"] } -serde = "1.0.147" -serde_derive = "1.0.147" -serde_json = "1.0.87" -log = "0.4.6" -env_logger = "0.9.1" -csv = "1.0.5" -ctrlc = "3.1.1" -crossbeam-utils = "0.8.12" +reqwest = { version = "0.12.8", features = [ + "blocking", + "json", + "rustls-tls", +], default-features = false } +serde = "1" +serde_derive = "1" +serde_json = "1" +csv = "1.3.0" + +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tokio = { version = "1.40", features = ["full", "tracing"] } +dotenvy = "0.15.7" +color-eyre = "0.6.3" +tracing = "0.1.40" +enum-map = { version = "2.7.3", features = ["serde"] } +thiserror = "1" + +console-subscriber = "0.4" diff --git a/src/config.rs b/src/config.rs index 9bac22dad2d..e579e5e15ac 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,27 +1,9 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - use std::path::PathBuf; +#[derive(Debug, Clone)] pub struct Config { - pub github_token: String, + // GitHub rate-limits are per-token, using more than one can speed up the scraper + pub github_token: Vec, pub data_dir: PathBuf, pub timeout: Option, } diff --git a/src/data.rs b/src/data.rs index 090ba1c056c..9eaab56aba8 100644 --- a/src/data.rs +++ b/src/data.rs @@ -1,41 +1,30 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -use config::Config; -use csv; -use prelude::*; -use serde_json; -use std::collections::HashMap; +use enum_map::{Enum, EnumMap}; +use serde_derive::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tokio::task::{spawn_blocking, JoinSet}; +use tracing::debug; + +use crate::config::Config; +use std::collections::BTreeMap; +use std::fs::OpenOptions; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::AtomicUsize; +use std::sync::Arc; use std::{ - fs::{self, File, OpenOptions}, + fs::{self, File}, io::{prelude::*, BufWriter}, }; -#[derive(Default, Serialize, Deserialize)] -struct State { - last_id: HashMap, +#[derive(Debug, Enum, Serialize, Deserialize, Copy, Clone)] +pub enum Forge { + #[serde(rename = "github")] + Github, } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Default, Serialize, Deserialize)] +struct State(EnumMap); + +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct Repo { pub id: String, pub name: String, @@ -43,76 +32,126 @@ pub struct Repo { pub has_cargo_lock: bool, } -pub struct Data { - base_dir: PathBuf, +#[derive(Debug)] +pub struct InnerData { + data_dir: PathBuf, + + state_lock: Mutex<()>, - csv_write_lock: Arc>, + state_cache: State, - state_path: PathBuf, - state_cache: Arc>>, + repos_state: Mutex>>, } +#[derive(Debug, Clone)] +pub struct Data(Arc); + impl Data { - pub fn new(config: &Config) -> Self { - Data { - base_dir: config.data_dir.clone(), + pub fn new(config: &Config) -> color_eyre::Result { + fs::create_dir_all(&config.data_dir)?; - csv_write_lock: Arc::new(Mutex::new(())), + let state_path = config.data_dir.join("state.json"); + let state_cache = if state_path.exists() { + serde_json::from_slice(&fs::read(&state_path)?)? + } else { + State::default() + }; - state_path: config.data_dir.join("state.json"), - state_cache: Arc::new(Mutex::new(None)), - } + let data = Self(Arc::new(InnerData { + data_dir: config.data_dir.clone(), + + state_lock: Mutex::new(()), + state_cache, + repos_state: Mutex::new(EnumMap::default()), + })); + + Ok(data) } - fn edit_state Fallible>(&self, f: F) -> Fallible { - let mut state_cache = self.state_cache.lock().unwrap(); + pub fn state_path(&self) -> PathBuf { + self.0.data_dir.join("state.json") + } - if state_cache.is_none() { - if self.state_path.exists() { - *state_cache = Some(serde_json::from_slice(&fs::read(&self.state_path)?)?); - } else { - *state_cache = Some(Default::default()); - } + pub fn csv_path(&self, forge: Forge) -> PathBuf { + match forge { + Forge::Github => self.0.data_dir.join("github.csv"), } + } - let state = state_cache.as_mut().unwrap(); - let result = f(state)?; + pub fn get_last_id(&self, forge: Forge) -> usize { + self.0.state_cache.0[forge].load(std::sync::atomic::Ordering::SeqCst) + } - let mut file = BufWriter::new(File::create(&self.state_path)?); - serde_json::to_writer_pretty(&mut file, &state)?; - file.write_all(&[b'\n'])?; + /// Store the state cache to disk, i.e. last fetched ids + async fn store_state_cache(&self) -> color_eyre::Result<()> { + let this = self.clone(); + let state_path = self.state_path(); + spawn_blocking(move || -> color_eyre::Result<()> { + let guard = this.0.state_lock.blocking_lock(); - Ok(result) - } + let file = File::create(state_path)?; + let mut file = BufWriter::new(file); + serde_json::to_writer_pretty(&mut file, &this.0.state_cache)?; + file.write_all(b"\n")?; - pub fn get_last_id(&self, platform: &str) -> Fallible> { - self.edit_state(|state| Ok(state.last_id.get(platform).cloned())) - } + drop(guard); - pub fn set_last_id(&self, platform: &str, id: usize) -> Fallible<()> { - self.edit_state(|state| { - state.last_id.insert(platform.to_string(), id); Ok(()) }) + .await + .unwrap() } - pub fn store_repo(&self, platform: &str, repo: Repo) -> Fallible<()> { - // Ensure only one thread can write to CSV files at once - let _lock = self.csv_write_lock.lock().unwrap(); + /// Stores the repos found to disk in a CSV + async fn store_csv(&self) -> color_eyre::Result<()> { + debug!("storing csv file"); + let mut repos = self.0.repos_state.lock().await; - let file = self.base_dir.join(format!("{}.csv", platform)); + let mut js = JoinSet::new(); - // Create the new file or append to it - let mut csv = if file.exists() { - csv::WriterBuilder::new() - .has_headers(false) - .from_writer(OpenOptions::new().append(true).open(&file)?) - } else { - csv::WriterBuilder::new().from_path(&file)? - }; + for (forge, repos) in repos.iter() { + let path = self.csv_path(forge); + let repos = repos.clone(); // is this necessary? + js.spawn_blocking(|| -> color_eyre::Result<()> { + let mut write_headers = false; + if !path.exists() { + File::create(&path)?; + write_headers = true; + } + + let file = OpenOptions::new().append(true).open(path)?; + + let mut writer = csv::WriterBuilder::new() + .has_headers(write_headers) + .from_writer(file); + + for (_, repo) in repos { + writer.serialize(repo)?; + } + + Ok(()) + }); + } + + js.join_all().await.into_iter().collect::>()?; - csv.serialize(repo)?; + // Clear the map + repos.iter_mut().for_each(|(_, m)| m.clear()); Ok(()) } + + pub async fn set_last_id(&self, forge: Forge, n: usize) -> color_eyre::Result<()> { + self.0.state_cache.0[forge].store(n, std::sync::atomic::Ordering::SeqCst); + + self.store_csv().await?; + self.store_state_cache().await?; + + Ok(()) + } + + pub async fn store_repo(&self, forge: Forge, repo: Repo) { + let mut repos_state = self.0.repos_state.lock().await; + repos_state[forge].insert(repo.name.clone(), repo); + } } diff --git a/src/github/api.rs b/src/github/api.rs index 13d7987a5dd..41b3dd70eac 100644 --- a/src/github/api.rs +++ b/src/github/api.rs @@ -1,320 +1,53 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -use config::Config; -use prelude::*; -use reqwest::blocking::{Client, RequestBuilder, Response}; -use reqwest::{header, Method, StatusCode}; -use serde::{de::DeserializeOwned, Serialize}; -use std::borrow::Cow; -use std::sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, +use std::{ + borrow::Cow, + future::Future, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, }; -use std::time::Duration; -static USER_AGENT: &str = "rust-repos (https://github.com/rust-ops/rust-repos)"; +use reqwest::{header, Client, Method, RequestBuilder, Response, StatusCode}; +use serde::de::DeserializeOwned; +use serde::ser::Serialize; +use serde_derive::Deserialize; +use serde_json::json; +use thiserror::Error; +use tokio::{task::yield_now, time::sleep}; +use tracing::{error, trace, warn}; -static GRAPHQL_QUERY_REPOSITORIES: &str = " -query($ids: [ID!]!) { - nodes(ids: $ids) { - ... on Repository { - id - nameWithOwner - defaultBranchRef { - name - } - languages(first: 100, orderBy: { field: SIZE, direction: DESC }) { - nodes { - name - } - } - } - } - - rateLimit { - cost - } -} -"; - -#[derive(Fail, Debug)] -#[fail(display = "internal github error: {:?}", _0)] -struct RetryRequest(StatusCode); - -trait ResponseExt { - fn handle_errors(self) -> Fallible - where - Self: Sized; -} +use crate::{config::Config, data::Repo}; -impl ResponseExt for Response { - fn handle_errors(self) -> Fallible { - let status = self.status(); - match status { - StatusCode::INTERNAL_SERVER_ERROR - | StatusCode::BAD_GATEWAY - | StatusCode::SERVICE_UNAVAILABLE - | StatusCode::GATEWAY_TIMEOUT => Err(RetryRequest(status).into()), - _ => Ok(self), - } - } -} +static USER_AGENT: &str = "rust-repos (https://github.com/rust-lang/rust-repos)"; -pub struct GitHubApi<'conf> { - config: &'conf Config, +#[derive(Debug)] +pub struct Github { client: Client, - slow_down: Arc, - concurrent_requests: Arc, + config: Config, + current_token_index: AtomicUsize, } -impl<'conf> GitHubApi<'conf> { - pub fn new(config: &'conf Config) -> Self { - GitHubApi { - config, - client: Client::new(), - slow_down: Arc::new(AtomicBool::new(false)), - concurrent_requests: Arc::new(AtomicUsize::new(0)), - } - } - - fn retry Fallible>(&self, f: F) -> Fallible { - let mut wait = Duration::from_secs(10); - let mut first = true; - - loop { - let concurrent = self.concurrent_requests.fetch_add(1, Ordering::SeqCst); - debug!( - "currently making {} concurrent requests to the GitHub API", - concurrent + 1 - ); - let res = f(); - self.concurrent_requests.fetch_sub(1, Ordering::SeqCst); - - match res { - Ok(res) => return Ok(res), - Err(err) => { - let mut retry = false; - if let Some(error) = err.downcast_ref::() { - warn!( - "API call to GitHub returned status code {}, retrying in {} seconds", - error.0, - wait.as_secs() - ); - retry = true; - } else if let Some(error) = err.downcast_ref::() { - if error.is_timeout() { - warn!( - "API call to GitHub timed out, retrying in {} seconds", - wait.as_secs() - ); - retry = true; - } - } else if let Some(error) = err.downcast_ref::() { - if error.kind() == std::io::ErrorKind::ConnectionReset { - warn!( - "connection to the API reset by peer, retrying in {} seconds", - wait.as_secs() - ); - retry = true; - } - } - - if !retry { - return Err(err); - } - } - } - - // Slow down only once per API call - if first { - self.slow_down.store(true, Ordering::SeqCst); - } - - ::std::thread::sleep(wait); - - // Stop doubling the time after a few increments, to avoid waiting too long - // This is still a request every ~10 minutes - if wait.as_secs() < 640 { - wait *= 2; - } - - first = false; - } - } - - fn build_request(&self, method: Method, url: &str) -> RequestBuilder { - let url = if !url.starts_with("https://") { - Cow::Owned(format!("https://api.github.com/{}", url)) - } else { - Cow::Borrowed(url) - }; - - self.client - .request(method, url.as_ref()) - .header( - header::AUTHORIZATION, - format!("token {}", self.config.github_token), - ) - .header(header::USER_AGENT, USER_AGENT) - } - - fn graphql(&self, query: &str, variables: V) -> Fallible { - self.retry(|| { - let resp: GraphResponse = self - .build_request(Method::POST, "graphql") - .json(&json!({ - "query": query, - "variables": variables, - })) - .send()? - .handle_errors()? - .json()?; - - if let Some(data) = resp.data { - if let Some(errors) = resp.errors { - for error in errors { - if let Some(ref type_) = error.type_ { - if type_ == "NOT_FOUND" { - debug!("ignored GraphQL error: {}", error.message); - continue; - } - } - - warn!("non-fatal GraphQL error: {}", error.message); - } - } - - Ok(data) - } else if let Some(mut errors) = resp.errors { - Err(err_msg(errors.pop().unwrap().message) - .context("GitHub GraphQL call failed") - .into()) - } else if let Some(message) = resp.message { - if message.contains("abuse") { - warn!("triggered GitHub abuse detection systems"); - Err(RetryRequest(StatusCode::TOO_MANY_REQUESTS).into()) - } else { - Err(err_msg(message) - .context("GitHub GraphQL call failed") - .into()) - } - } else { - Err(err_msg("empty GraphQL response")) - } - }) - } - - pub fn scrape_repositories(&self, since: usize) -> Fallible>> { - self.retry(|| { - let resp = self - .build_request(Method::GET, &format!("repositories?since={}", since)) - .send()? - .handle_errors()?; - - let status = resp.status(); - if status == StatusCode::OK { - Ok(resp.json()?) - } else { - let error: GitHubError = resp.json()?; - if error.message.contains("abuse") { - warn!("triggered GitHub abuse detection systems"); - Err(RetryRequest(StatusCode::TOO_MANY_REQUESTS).into()) - } else { - Err(err_msg(error.message) - .context(format!( - "GitHub API call failed with status code: {}", - status - )) - .context(format!( - "failed to fetch GitHub repositories since ID {}", - since - )) - .into()) - } - } - }) - } - - pub fn load_repositories(&self, node_ids: &[String]) -> Fallible>> { - let data: GraphRepositories = self.graphql( - GRAPHQL_QUERY_REPOSITORIES, - json!({ - "ids": node_ids, - }), - )?; - - assert!( - data.rate_limit.cost <= 1, - "load repositories query too costly" - ); - Ok(data.nodes) - } - - pub fn file_exists(&self, repo: &GraphRepository, path: &str) -> Fallible { - let url = format!( - "https://raw.githubusercontent.com/{}/{}/{}", - repo.name_with_owner, - if let Some(ref_) = &repo.default_branch_ref { - &ref_.name - } else { - "master" - }, - path, - ); +#[derive(Debug, Deserialize)] +pub struct GitHubError { + message: String, - self.retry(|| { - let resp = self - .build_request(Method::GET, &url) - .send()? - .handle_errors()?; - match resp.status() { - StatusCode::OK => Ok(true), - StatusCode::NOT_FOUND => Ok(false), - status => Err( - err_msg(format!("GitHub API returned status code {}", status)) - .context(format!( - "failed to fetch file {} from repo {}", - path, repo.name_with_owner, - )) - .into(), - ), - } - }) - } + #[allow(unused)] + r#type: Option, +} - pub fn should_slow_down(&self) -> bool { - self.slow_down.swap(false, Ordering::SeqCst) - } +#[derive(Clone, Debug, Deserialize)] +pub struct Node { + pub path: String, } -#[derive(Deserialize)] -struct GitHubError { - message: String, - #[serde(rename = "type")] - type_: Option, +#[derive(Debug, Deserialize)] +pub struct GithubTree { + pub tree: Vec, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub struct RestRepository { pub id: usize, + // Useful for debugging, if something does go wrong + #[allow(unused)] pub full_name: String, pub node_id: String, pub fork: bool, @@ -324,6 +57,7 @@ pub struct RestRepository { struct GraphResponse { data: Option, errors: Option>, + #[allow(unused)] message: Option, } @@ -344,10 +78,20 @@ struct GraphRepositories { pub struct GraphRepository { pub id: String, pub name_with_owner: String, - pub default_branch_ref: Option, pub languages: GraphLanguages, } +impl GraphRepository { + pub fn into_repo(self, has_cargo_toml: bool, has_cargo_lock: bool) -> Repo { + Repo { + id: self.id, + name: self.name_with_owner, + has_cargo_toml, + has_cargo_lock, + } + } +} + #[derive(Debug, Deserialize)] pub struct GraphLanguages { pub nodes: Vec>, @@ -358,14 +102,241 @@ pub struct GraphLanguage { pub name: String, } -#[derive(Debug, Deserialize)] -pub struct GraphRef { - pub name: String, +#[derive(Debug, Error)] +pub enum Error { + #[error("reqwest error occurred {0:?}")] + Reqwest(#[from] reqwest::Error), + #[error("rate limit hit {0}")] + RateLimit(StatusCode), + #[error("other http error: {0}")] + HttpStatus(StatusCode), + + #[error("Response did not contain requested data")] + EmptyData, + #[error("IO Error {0}")] + Io(#[from] std::io::Error), } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum GitHubErrorType { - NotFound, - Other(String), +/// 100 is the max results per page of the GH API +pub(crate) const N: usize = 100; + +const GRAPHQL_QUERY_REPOSITORIES: &str = " +query($ids: [ID!]!) { + nodes(ids: $ids) { + ... on Repository { + id + nameWithOwner + languages(first: 100, orderBy: { field: SIZE, direction: DESC }) { + nodes { + name + } + } + } + } + + rateLimit { + cost + } +} +"; + +impl Github { + pub fn new(config: Config) -> Self { + Github { + client: Client::new(), + current_token_index: AtomicUsize::new(0), + config, + } + } + + #[inline] + fn get_token(&self) -> &str { + &self.config.github_token[self.current_token_index.load(Ordering::Relaxed)] + } + + fn build_request(&self, method: Method, url: &str) -> RequestBuilder { + let url = if url.starts_with("https://") { + Cow::from(url) + } else { + Cow::from(format!("https://api.github.com/{url}")) + }; + trace!("Sending request to {url}"); + self.client + .request(method, url.as_ref()) + .header(header::AUTHORIZATION, format!("token {}", self.get_token())) + .header(header::USER_AGENT, USER_AGENT) + // .header(header::ACCEPT, "application/vnd.github+json") + } + + async fn graphql( + &self, + query: &str, + variables: S, + ) -> Result { + let resp = self + .build_request(Method::POST, "graphql") + .json(&json!({ + "query": query, + "variables": variables, + })) + .send() + .await?; + + let data: GraphResponse = handle_response_json(resp).await?; + + if let Some(errs) = data.errors { + let errs: Vec<_> = errs + .into_iter() + // Some repos on GitHub are just marked as NOT_FOUND, does not seem like our fault + .filter(|el| el.r#type.as_deref() != Some("NOT_FOUND")) + .collect(); + if !errs.is_empty() { + warn!("GraphQL Errors: \n {:#?}", errs); + } + } + + data.data.ok_or_else(|| Error::EmptyData) + } + + pub async fn load_repositories( + &self, + node_ids: &[String], + ) -> Result, Error> { + let data: GraphRepositories = self + .retry(|| async { + self.graphql( + GRAPHQL_QUERY_REPOSITORIES, + json!({ + "ids": node_ids, + }), + ) + .await + }) + .await?; + + assert!( + data.rate_limit.cost <= 1, + "load repositories query too costly" + ); + + Ok(data.nodes.into_iter().flatten().collect()) + } + + /// gets a file tree of a specific github repo + pub async fn tree(&self, repo: &Repo, recursive: bool) -> Result { + let mut url = format!("repos/{}/git/trees/HEAD", repo.name); + if recursive { + url = format!("{url}?recursive=1"); + } + + self.retry(|| async { + let resp = self.build_request(Method::GET, &url).send().await?; + + handle_response_json(resp).await + }) + .await + } + + /// scrapes all github repos (paginated) + pub async fn scrape_repositories(&self, since: usize) -> Result, Error> { + // Maybe needs to be a Vec> + let output: Vec = self + .retry(|| async { + let resp = self + .build_request( + Method::GET, + &format!("repositories?since={since}&per_page{N}"), + ) + .send() + .await?; + + handle_response_json(resp).await + }) + .await?; + + if output.len() != N { + warn!("Github API returned {} instead of {N} repos", output.len()); + } + + Ok(output) + } + + /// retry a github api request and rotate tokens to circumvent rate limiting + /// On reqwest errors does exponential backoff until 5 mins. + async fn retry(&self, fun: F) -> Result + where + F: Fn() -> Fu, + Fu: Future>, + { + let mut backoff = Duration::from_secs(1); + loop { + match fun().await { + ok @ Ok(_) => return ok, + Err(Error::Reqwest(reqwest_error)) => { + warn!("Reqwest encountered error {reqwest_error:?}"); + warn!("Backing off for {} seconds", backoff.as_secs()); + sleep(backoff).await; + + backoff = backoff + backoff + Duration::from_millis(123); // Exponential backoff + jitter + + // After 5 minutes bail + if backoff.as_secs() > 300 { + error!("Failed sending request 5 times"); + return Err(Error::Reqwest(reqwest_error)); + } + } + Err(err @ Error::HttpStatus(_)) => return Err(err), + Err(Error::RateLimit(_)) => { + let mut wait = false; + self.current_token_index + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |old| { + if old + 1 >= self.config.github_token.len() { + wait = true; + Some(0) + } else { + Some(old + 1) + } + }) + .unwrap(); + + if wait { + warn!("Tokens wrapped around, sleeping for 1 minute"); + sleep(Duration::from_secs(60)).await; + } + } + err @ Err(_) => return err, + } + + // Yield + yield_now().await; + } + } +} + +async fn handle_response_json(resp: Response) -> Result { + let res = handle_response(resp).await?.json().await?; + Ok(res) +} + +/// Converts github responses into the correct error codes (helper for the retry function) +async fn handle_response(resp: Response) -> Result { + let status = resp.status(); + if status.is_success() { + Ok(resp) + } else if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::UNPROCESSABLE_ENTITY + { + warn!("Rate limit hit"); + Err(Error::RateLimit(status)) + } else if let Ok(error) = resp.json().await { + let error: GitHubError = error; + if error.message.contains("abuse") || error.message.contains("rate limit") { + warn!("Rate limit hit ({}): {}", status.as_u16(), error.message); + Err(Error::RateLimit(status)) + } else { + warn!("Http Error ({}): {}", status.as_u16(), error.message); + Err(Error::HttpStatus(status)) + } + } else { + Err(Error::HttpStatus(status)) + } } diff --git a/src/github/mod.rs b/src/github/mod.rs index c09a7119e88..c8609105b90 100644 --- a/src/github/mod.rs +++ b/src/github/mod.rs @@ -1,111 +1,116 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +use api::Github; +use tokio::{ + signal::ctrl_c, + task::JoinSet, + time::{sleep, Instant}, +}; +use tracing::{debug, info, warn}; + +use crate::{ + config::Config, + data::{Data, Forge}, +}; mod api; -use config::Config; -use crossbeam_utils::thread::scope; -use data::{Data, Repo}; -use github::api::GitHubApi; -use prelude::*; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; -use utils::wrap_thread; - -static WANTED_LANG: &str = "Rust"; - -fn load_thread(api: &GitHubApi, data: &Data, to_load: Vec) -> Fallible<()> { - debug!( - "collected {} non-fork repositories, loading them", - to_load.len() - ); - - let mut graph_repos = api.load_repositories(&to_load)?; - for repo in graph_repos.drain(..).flatten() { - let mut found = false; - for lang in repo.languages.nodes.iter().filter_map(Option::as_ref) { - if lang.name == WANTED_LANG { - found = true; - break; - } - } +#[derive(Debug, Clone)] +pub struct Scraper { + gh: Arc, + data: Data, + finished: Arc, + timeout: Option, +} - if found { - let has_cargo_toml = api.file_exists(&repo, "Cargo.toml")?; - let has_cargo_lock = api.file_exists(&repo, "Cargo.lock")?; - - data.store_repo( - "github", - Repo { - id: repo.id, - name: repo.name_with_owner.clone(), - has_cargo_toml, - has_cargo_lock, - }, - )?; - - info!( - "found {}: Cargo.toml = {:?}, Cargo.lock = {:?}", - repo.name_with_owner, has_cargo_toml, has_cargo_lock, - ); +impl Scraper { + pub fn new(config: Config, data: Data) -> Self { + let timeout = config.timeout; + let gh = Github::new(config); + + let finished = Arc::new(AtomicBool::new(false)); + let f2 = finished.clone(); + + tokio::spawn(async move { + ctrl_c().await.expect("Failed to install Ctrl+C Handler"); + warn!("Ctrl+C received, stopping..."); + f2.store(true, Ordering::SeqCst); + }); + + Self { + gh: Arc::new(gh), + data, + finished, + timeout, } } - // Applease Clippy - ::std::mem::drop(to_load); + async fn load_repositories(&self, repos: Vec) -> color_eyre::Result<()> { + let mut graph_repos = self.gh.load_repositories(&repos).await?; + let mut js = JoinSet::new(); + for repo in graph_repos.drain(..) { + if repo + .languages + .nodes + .iter() + .filter_map(Option::as_ref) + .any(|el| el.name == "Rust") + { + let this = self.clone(); + js.spawn(async move { + let mut repo = repo.into_repo(false, false); + let files = this.gh.tree(&repo, false).await; + match files { + Ok(tree) => { + for node in tree.tree { + if node.path == "Cargo.toml" { + repo.has_cargo_toml = true; + } else if node.path == "Cargo.lock" { + repo.has_cargo_lock = true; + } + } + } + Err(e) => { + warn!("Could not get tree for {}, error: {e:?}", repo.name); + } + } + + this.data.store_repo(Forge::Github, repo).await; + }); + } + } - Ok(()) -} + js.join_all().await; -pub fn scrape(data: &Data, config: &Config, should_stop: &AtomicBool) -> Fallible<()> { - info!("started scraping for GitHub repositories"); + Ok(()) + } - let gh = api::GitHubApi::new(config); - let mut to_load = Vec::with_capacity(100); + pub async fn scrape(&self) -> color_eyre::Result<()> { + let start = Instant::now(); - let result = scope(|scope| { - let mut last_id = data.get_last_id("github")?.unwrap_or(0); - let scrape_start = Instant::now(); + let mut to_load = Vec::with_capacity(api::N); + + let mut last_id = self.data.get_last_id(Forge::Github); loop { - if let Some(timeout) = config.timeout { - if scrape_start.elapsed() >= Duration::from_secs(timeout) { - info!("timeout reached, stopping the scraping loop"); + let start_loop = Instant::now(); + if let Some(timeout) = self.timeout { + if start.elapsed() >= Duration::from_secs(timeout) { + info!("Timeout reached, stopped scraping"); break; } } - // Wait 2 minutes if GitHub is slowing us down - if gh.should_slow_down() { - warn!("slowing down the scraping (2 minutes pause)"); - ::std::thread::sleep(Duration::from_secs(120)); - } - - let start = Instant::now(); - - debug!("scraping 100 repositories from the REST API"); + let mut repos = self.gh.scrape_repositories(last_id).await?; + let mut js = JoinSet::new(); - // Load all the non-fork repositories in the to_load vector - let mut repos = gh.scrape_repositories(last_id)?; - let finished = repos.len() < 100 || should_stop.load(Ordering::SeqCst); - for repo in repos.drain(..).flatten() { + for repo in repos.drain(..) { last_id = repo.id; if repo.fork { continue; @@ -113,35 +118,38 @@ pub fn scrape(data: &Data, config: &Config, should_stop: &AtomicBool) -> Fallibl to_load.push(repo.node_id); - if to_load.len() == 100 { + if to_load.len() == api::N { let to_load_now = to_load.clone(); - scope.spawn(|_| wrap_thread(|| load_thread(&gh, data, to_load_now))); + let this = self.clone(); + js.spawn(async move { this.load_repositories(to_load_now).await }); to_load.clear(); } } - data.set_last_id("github", last_id)?; + self.data.set_last_id(Forge::Github, last_id).await?; - if finished { - // Ensure all the remaining repositories are loaded - if !to_load.is_empty() { - let to_load_now = to_load.clone(); - scope.spawn(|_| wrap_thread(|| load_thread(&gh, data, to_load_now))); + for res in js.join_all().await { + if let Err(e) = res { + warn!("Failed scraping repo: {:?}", e); } + } + if self.finished.load(Ordering::SeqCst) { + if !to_load.is_empty() { + self.load_repositories(to_load).await?; + } break; } - // Avoid hammering GitHub too much - if let Some(sleep) = Duration::from_secs(1).checked_sub(start.elapsed()) { - ::std::thread::sleep(sleep); + debug!("Loaded 100 repos in {}ms", start_loop.elapsed().as_millis()); + + if let Some(time) = Duration::from_millis(250).checked_sub(start_loop.elapsed()) { + sleep(time).await; } } - Ok(()) - }) - .unwrap(); + info!("Took {} seconds", start.elapsed().as_secs()); - info!("finished scraping for GitHub repositories"); - result + Ok(()) + } } diff --git a/src/main.rs b/src/main.rs index c0f45ed852b..2d262c607af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,129 +1,58 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. +use std::{env, path::PathBuf}; -extern crate crossbeam_utils; -extern crate csv; -extern crate ctrlc; -extern crate env_logger; -#[macro_use] -extern crate failure; -#[macro_use] -extern crate log; -extern crate reqwest; -extern crate serde; -#[macro_use] -extern crate serde_derive; -#[macro_use] -extern crate serde_json; +use config::Config; +use data::Data; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; mod config; mod data; mod github; -mod prelude; -mod utils; -use config::Config; -use prelude::*; -use std::path::PathBuf; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; -use std::time::Instant; +fn get_tokens_from_env() -> Vec { + let env_token = env::var("GITHUB_TOKEN").ok(); + let env_tokens = env::var("GITHUB_TOKENS").ok(); -fn app() -> Fallible<()> { - // Get the GitHub token from the environment - let github_token = - std::env::var("GITHUB_TOKEN").context("failed to get the GitHub API token")?; + let mut tokens = vec![]; - let timeout = if let Ok(var) = std::env::var("RUST_REPOS_TIMEOUT") { - Some( - var.parse::() - .context("failed to parse RUST_REPOS_TIMEOUT")?, - ) - } else { - None - }; - - // Parse CLI arguments - let args = std::env::args().skip(1).collect::>(); - if args.is_empty() { - bail!("missing argument: "); - } else if args.len() > 1 { - bail!("too many arguments"); + if let Some(t) = env_token { + tokens.push(t); } - // Ensure the data directory exists - let data_dir = PathBuf::from(&args[0]); - if !data_dir.is_dir() { - debug!( - "created missing data directory: {}", - data_dir.to_string_lossy() - ); - std::fs::create_dir_all(&data_dir)?; + if let Some(ts) = env_tokens { + let ts = ts.split(','); + for t in ts { + tokens.push(t.to_string()); + } } - let config = Config { - github_token, - data_dir, - timeout, - }; - - let data = data::Data::new(&config); + tokens +} - let should_stop = Arc::new(AtomicBool::new(false)); - let stop = should_stop.clone(); - ctrlc::set_handler(move || { - info!("received Ctrl+C, terminating..."); - stop.store(true, Ordering::SeqCst); - })?; +#[tokio::main] +async fn main() -> color_eyre::Result<()> { + dotenvy::dotenv().ok(); + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_default_env()) + .init(); - github::scrape(&data, &config, &should_stop)?; + let github_token = get_tokens_from_env(); - Ok(()) -} + let timeout = env::var("RUST_REPOS_TIMEOUT") + .ok() + .and_then(|el| el.parse::().ok()); -fn main() { - // Initialize logging - // This doesn't use from_default_env() because it doesn't allow to override filter_module() - // with the RUST_LOG environment variable - let mut logger = env_logger::Builder::new(); - logger.filter_module("rust_repos", log::LevelFilter::Info); - if let Ok(content) = std::env::var("RUST_LOG") { - logger.parse_filters(&content); - } - logger.init(); + let data_dir = PathBuf::from("./data_new"); - let start = Instant::now(); + let config = Config { + github_token, + data_dir, + timeout, + }; - let result = app(); - if let Err(ref err) = &result { - utils::log_error(err); - } + let data = Data::new(&config)?; - info!( - "execution completed in {} seconds", - start.elapsed().as_secs() - ); + let scraper = github::Scraper::new(config, data); - if result.is_err() { - std::process::exit(1); - } + scraper.scrape().await } diff --git a/src/prelude.rs b/src/prelude.rs deleted file mode 100644 index 57bc2552e2e..00000000000 --- a/src/prelude.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -pub use failure::{err_msg, Error, Fail, Fallible, ResultExt}; diff --git a/src/utils.rs b/src/utils.rs deleted file mode 100644 index 074fbd9cc57..00000000000 --- a/src/utils.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2018 Pietro Albini -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of -// this software and associated documentation files (the "Software"), to deal in -// the Software without restriction, including without limitation the rights to -// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -// of the Software, and to permit persons to whom the Software is furnished to do -// so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -use prelude::*; - -pub fn log_error(err: &Error) { - error!("{}", err); - for cause in err.iter_causes() { - error!(" caused by: {}", cause); - } -} - -pub fn wrap_thread Fallible<()>>(f: F) { - if let Err(err) = f() { - log_error(&err); - } -}