diff --git a/Cargo.lock b/Cargo.lock index 45685a141d9..f90f451d8ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "color-eyre" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + [[package]] name = "colorchoice" version = "1.0.2" @@ -190,6 +217,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", + "serde", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "env_filter" version = "0.1.2" @@ -213,6 +267,16 @@ dependencies = [ "log", ] +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "failure" version = "0.1.8" @@ -231,7 +295,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2", "quote", - "syn 1.0.102", + "syn 1.0.109", "synstructure", ] @@ -435,6 +499,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "ipnet" version = "2.5.0" @@ -462,12 +532,28 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.22" @@ -519,6 +605,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "object" version = "0.29.0" @@ -534,6 +630,41 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -657,6 +788,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.6.0" @@ -736,9 +876,12 @@ dependencies = [ name = "rust-repos" version = "0.1.0" dependencies = [ + "color-eyre", "crossbeam-utils", "csv", "ctrlc", + "dotenvy", + "enum-map", "env_logger", "failure", "log", @@ -746,6 +889,9 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -806,6 +952,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.210" @@ -850,12 +1002,30 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.7" @@ -895,9 +1065,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.102" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -932,7 +1102,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2", "quote", - "syn 1.0.102", + "syn 1.0.109", "unicode-xid", ] @@ -956,6 +1126,16 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -981,11 +1161,26 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", + "tracing", "windows-sys 0.52.0", ] +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -1005,22 +1200,69 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -1052,9 +1294,9 @@ dependencies = [ [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -1079,6 +1321,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "want" version = "0.3.0" @@ -1181,6 +1429,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-registry" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 45fb3282794..de7e6e617f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,17 @@ authors = ["Pietro Albini "] failure = "0.1.8" reqwest = { version = "0.12.8", features = ["blocking", "json", "rustls-tls"], default-features = false } serde = "1.0.210" -serde_derive = "1.0.210" -serde_json = "1.0.128" +serde_derive = "1" +serde_json = "1" log = "0.4.22" env_logger = "0.11.5" csv = "1.3.0" ctrlc = "3.4.5" crossbeam-utils = "0.8.20" + +tokio = { version = "1.40", features = ["full", "tracing"] } +dotenvy = "0.15.7" +color-eyre = "0.6.3" +tracing = "0.1.40" +tracing-subscriber = "0.3" +enum-map = { version = "2.7.3", features = ["serde"] } diff --git a/src/data.rs b/src/data.rs index 858be0195bd..711e9e45595 100644 --- a/src/data.rs +++ b/src/data.rs @@ -18,22 +18,31 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +use enum_map::{Enum, EnumMap}; +use tokio::sync::Mutex; +use tokio::task::{spawn_blocking, JoinSet}; + use crate::config::Config; -use crate::prelude::*; -use std::collections::HashMap; +use std::collections::BTreeMap; +use std::fs::OpenOptions; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::AtomicUsize; +use std::sync::Arc; +use std::time::Instant; use std::{ - fs::{self, File, OpenOptions}, + fs::{self, File}, io::{prelude::*, BufWriter}, }; -#[derive(Default, Serialize, Deserialize)] -struct State { - last_id: HashMap, +#[derive(Enum, Serialize, Deserialize, Copy, Clone)] +enum Forge { + Github, } -#[derive(Serialize, Deserialize)] +#[derive(Default, Serialize, Deserialize)] +struct State(EnumMap); + +#[derive(Serialize, Deserialize, Clone)] pub struct Repo { pub id: String, pub name: String, @@ -42,75 +51,130 @@ pub struct Repo { } pub struct Data { - base_dir: PathBuf, + data_dir: PathBuf, csv_write_lock: Arc>, + state_lock: Arc>, - state_path: PathBuf, - state_cache: Arc>>, + state_cache: Arc, + + last_store: Instant, + repos_state: Arc>>>, } impl Data { - pub fn new(config: &Config) -> Self { - Data { - base_dir: config.data_dir.clone(), + pub fn new(config: &Config) -> color_eyre::Result { + let mut data = Data { + data_dir: config.data_dir.clone(), csv_write_lock: Arc::new(Mutex::new(())), - state_path: config.data_dir.join("state.json"), - state_cache: Arc::new(Mutex::new(None)), - } - } + state_lock: Arc::new(Mutex::new(())), + state_cache: Arc::new(State::default()), + repos_state: Arc::new(Mutex::new(EnumMap::default())), + last_store: Instant::now(), + }; - fn edit_state Fallible>(&self, f: F) -> Fallible { - let mut state_cache = self.state_cache.lock().unwrap(); + // TODO: create CSV files if not exist - if state_cache.is_none() { - if self.state_path.exists() { - *state_cache = Some(serde_json::from_slice(&fs::read(&self.state_path)?)?); - } else { - *state_cache = Some(Default::default()); - } + + let state_path = data.state_path(); + if state_path.exists() { + let state_cache: State = serde_json::from_slice(&fs::read(&state_path)?)?; + + data.state_cache = Arc::new(state_cache) } - let state = state_cache.as_mut().unwrap(); - let result = f(state)?; + Ok(data) + } - let mut file = BufWriter::new(File::create(&self.state_path)?); - serde_json::to_writer_pretty(&mut file, &state)?; - file.write_all(&[b'\n'])?; + pub fn state_path(&self) -> PathBuf { + self.data_dir.join("state.json") + } - Ok(result) + pub fn csv_path(&self, forge: Forge) -> PathBuf { + match forge { + Forge::Github => self.data_dir.join("github"), + } } - pub fn get_last_id(&self, platform: &str) -> Fallible> { - self.edit_state(|state| Ok(state.last_id.get(platform).cloned())) + pub fn get_last_ids(&self) -> Arc { + self.state_cache.clone() } - pub fn set_last_id(&self, platform: &str, id: usize) -> Fallible<()> { - self.edit_state(|state| { - state.last_id.insert(platform.to_string(), id); + /// Store the state cache to disk, i.e. last fetched ids + async fn store_state_cache(&self) -> color_eyre::Result<()> { + let state = self.state_cache.clone(); + let lock = self.state_lock.clone(); + let state_path = self.state_path(); + spawn_blocking(move || -> color_eyre::Result<()> { + let guard = lock.blocking_lock(); + + let file = File::create(state_path)?; + let mut file = BufWriter::new(file); + serde_json::to_writer_pretty(&mut file, state.as_ref()); + file.write_all(b"\n")?; + + drop(guard); + Ok(()) }) + .await + .unwrap() } - pub fn store_repo(&self, platform: &str, repo: Repo) -> Fallible<()> { - // Ensure only one thread can write to CSV files at once - let _lock = self.csv_write_lock.lock().unwrap(); + /// Stores the repos found to disk in a CSV + async fn store_csv(&self) -> color_eyre::Result<()> { + let mut repos = self.repos_state.lock().await; - let file = self.base_dir.join(format!("{}.csv", platform)); + let mut js = JoinSet::new(); - // Create the new file or append to it - let mut csv = if file.exists() { - csv::WriterBuilder::new() - .has_headers(false) - .from_writer(OpenOptions::new().append(true).open(&file)?) - } else { - csv::WriterBuilder::new().from_path(&file)? - }; + for (forge, repos) in repos.iter() { + let path = self.csv_path(forge); + let repos = repos.clone(); // is this necessary? + js.spawn_blocking(|| -> color_eyre::Result<()> { + let mut write_headers = false; + if !path.exists() { + File::create(&path)?; + write_headers = true; + } + + let file = OpenOptions::new() + .append(true) + .open(path)?; + + let mut writer = csv::WriterBuilder::new() + .has_headers(write_headers) + .from_writer(file); + + for (_, repo) in repos { + writer.serialize(repo)?; + } - csv.serialize(repo)?; + Ok(()) + }); + } + + js.join_all().await.into_iter().collect::>()?; + + // Clear the map + repos.iter_mut().for_each(|(_, m)| m.clear()); Ok(()) } + + pub async fn set_last_id(&self, forge: Forge, n: usize) -> color_eyre::Result<()> { + self.state_cache.0[forge].store(n, std::sync::atomic::Ordering::SeqCst); + + self.store_csv().await?; + self.store_state_cache().await?; + + Ok(()) + } + + pub async fn store_repo(&self, forge: Forge, repo: Repo) { + let mut repos_state = self.repos_state.lock().await; + repos_state[forge].insert(repo.name.clone(), repo); + } } + diff --git a/src/main.rs b/src/main.rs index c0f45ed852b..cab32310492 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,7 +100,7 @@ fn app() -> Fallible<()> { Ok(()) } -fn main() { +fn main2() { // Initialize logging // This doesn't use from_default_env() because it doesn't allow to override filter_module() // with the RUST_LOG environment variable @@ -127,3 +127,13 @@ fn main() { std::process::exit(1); } } + + +#[tokio::main] +async fn main() -> color_eyre::Result<()> { + let tokens = todo!(); + + + + Ok(()) +} \ No newline at end of file