From d5d6a44377d823a91438dfd588dea1c55a8f0ae0 Mon Sep 17 00:00:00 2001 From: jrx Date: Sun, 28 Apr 2024 01:21:06 +0200 Subject: [PATCH] csv --- Cargo.lock | 236 +++++++++++++++++++++----- codegenr/Cargo.toml | 11 +- codegenr/_samples/username.csv | 7 + codegenr/src/loaders/csv.rs | 24 +++ codegenr/src/loaders/document_path.rs | 3 + codegenr/src/loaders/mod.rs | 27 ++- codegenr/src/loaders/xml.rs | 2 +- codegenr/src/processor/file.rs | 2 +- 8 files changed, 256 insertions(+), 56 deletions(-) create mode 100644 codegenr/_samples/username.csv create mode 100644 codegenr/src/loaders/csv.rs diff --git a/Cargo.lock b/Cargo.lock index 6743a5d..f65ea54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,7 +77,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" dependencies = [ "flate2", - "http", + "http 0.2.9", "log", "rustls", "url", @@ -119,9 +119,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" [[package]] name = "bitflags" @@ -194,9 +194,10 @@ dependencies = [ [[package]] name = "codegenr" -version = "0.0.4" +version = "0.0.5" dependencies = [ "anyhow", + "csv", "glob", "graphql-parser", "handlebars", @@ -308,6 +309,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "deunicode" version = "0.4.4" @@ -434,6 +456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -468,6 +491,7 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", "futures-io", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -520,17 +544,17 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", - "indexmap 1.9.3", + "http 1.1.0", + "indexmap 2.0.0", "slab", "tokio", "tokio-util", @@ -620,14 +644,37 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" -version = "0.4.5" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body", "pin-project-lite", ] @@ -637,47 +684,60 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hyper" -version = "0.14.27" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", - "http", + "http 1.1.0", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", + "http-body-util", "hyper", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1015,6 +1075,26 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1093,11 +1173,12 @@ dependencies = [ [[package]] name = "quickxml_to_serde" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f35112b35480fd72f63444289083eeedbd61d13907c82c4309f0ccda35e244" +checksum = "286b05c7a00b356ff6ac5218e10d628e0a3be02e777d067ca7286d353c3c407e" dependencies = [ "minidom", + "regex", "serde", "serde_derive", "serde_json", @@ -1198,20 +1279,23 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", - "http", + "http 1.1.0", "http-body", + "http-body-util", "hyper", "hyper-tls", + "hyper-util", "ipnet", "js-sys", "log", @@ -1220,9 +1304,12 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", + "system-configuration", "tokio", "tokio-native-tls", "tower-service", @@ -1307,6 +1394,22 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" + [[package]] name = "ryu" version = "1.0.15" @@ -1482,9 +1585,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" dependencies = [ "serde", ] @@ -1501,16 +1604,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.4" @@ -1585,6 +1678,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempdir" version = "0.3.7" @@ -1718,7 +1838,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.4", + "socket2", "windows-sys", ] @@ -1806,6 +1926,28 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -2212,9 +2354,9 @@ dependencies = [ [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys", diff --git a/codegenr/Cargo.toml b/codegenr/Cargo.toml index 3223100..e1ffbbc 100644 --- a/codegenr/Cargo.toml +++ b/codegenr/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codegenr" -version = "0.0.4" +version = "0.0.5" edition = "2021" authors = ["Jérôme Rx & contributors "] description = "Fast json/yaml/openapi code generator based on handlebars templating." @@ -35,6 +35,7 @@ structopt = { version = "0.3", optional = true } anyhow = { version = "1.0", optional = true } toml = { version = "0.8", optional = true } + # ======================================================================================== # Common dependencies # ======================================================================================== @@ -45,17 +46,19 @@ thiserror = "1.0" # File loading & path manipulations url = "2.4" path-dedot = "3.1" -reqwest = { version = "0.11", features = ["blocking"] } +reqwest = { version = "0.12", features = ["blocking"] } walkdir = "2" glob = "0.3" -# Json / Yaml / OpenApi / Graphql manipulation + +# file formats manipulation serde_json = { version = "1.0", features = ["preserve_order"] } serde_yaml = "0.8" # 0.9 failing with deserializing from YAML containing more than one document is not supported on most files serde = "1.0" graphql-parser = "0.4" quick-xml = { version = "0.31", features = ["serialize"] } -quickxml_to_serde = "0.5" +quickxml_to_serde = "0.6" minidom = "0.12" +csv = "1.2" # Templating handlebars = { version = "4.4", features = ["script_helper"] } diff --git a/codegenr/_samples/username.csv b/codegenr/_samples/username.csv new file mode 100644 index 0000000..402a348 --- /dev/null +++ b/codegenr/_samples/username.csv @@ -0,0 +1,7 @@ +Username, Identifier,First name,Last name +booker12,9012,Rachel,Booker +grey07,2070,Laura,Grey +johnson81,4081,Craig,Johnson +jenkins46,9346,Mary,Jenkins +smith79,5079,Jamie,Smith + diff --git a/codegenr/src/loaders/csv.rs b/codegenr/src/loaders/csv.rs new file mode 100644 index 0000000..72a1e3b --- /dev/null +++ b/codegenr/src/loaders/csv.rs @@ -0,0 +1,24 @@ +use super::DocumentLoader; +use serde_json::Value; + +pub struct CsvLoader {} +impl DocumentLoader for CsvLoader { + type Error = csv::Error; + fn json_from_str(content: &str) -> Result { + let mut rdr = csv::Reader::from_reader(content.as_bytes()); + let lines = rdr + .records() + .map(|result| { + let record = result?; + Ok(Value::Object( + record + .into_iter() + .enumerate() + .map(|(i, field)| (format!("field_{i}"), Value::String(field.into()))) + .collect(), + )) + }) + .collect::, Self::Error>>()?; + Ok(Value::Array(lines)) + } +} diff --git a/codegenr/src/loaders/document_path.rs b/codegenr/src/loaders/document_path.rs index 8bfbc1e..8073cfa 100644 --- a/codegenr/src/loaders/document_path.rs +++ b/codegenr/src/loaders/document_path.rs @@ -74,6 +74,7 @@ impl DocumentPath { DocumentPath::FileName(s) => s, DocumentPath::None => return FormatHint::NoIdea, }; + let s = s.to_lowercase(); if s.ends_with(".json") { FormatHint::Json } else if s.ends_with(".yaml") || s.ends_with(".yml") { @@ -84,6 +85,8 @@ impl DocumentPath { FormatHint::Graphql } else if s.ends_with(".xml") || s.ends_with(".xaml") || s.ends_with(".wsdl") || s.ends_with(".xsd") || s.ends_with(".xul") { FormatHint::Xml + } else if s.ends_with(".csv") { + FormatHint::Csv } else { FormatHint::NoIdea } diff --git a/codegenr/src/loaders/mod.rs b/codegenr/src/loaders/mod.rs index 271fe37..2af253c 100644 --- a/codegenr/src/loaders/mod.rs +++ b/codegenr/src/loaders/mod.rs @@ -4,6 +4,7 @@ use url::Url; pub mod document_path; pub use document_path::*; +pub mod csv; pub mod graphql; pub mod json; pub mod toml; @@ -50,6 +51,7 @@ pub enum LoaderError { yaml_error: serde_yaml::Error, toml_error: ::toml::de::Error, xml_error: minidom::Error, + csv_error: ::csv::Error, graphql_error: graphql_parser::schema::ParseError, }, #[error("Yaml error: `{0}`.")] @@ -74,6 +76,8 @@ pub(crate) enum FormatHint { Toml, /// The content should be xml Xml, + /// The content should be csv + Csv, /// The content should be a graphql schema Graphql, /// We have no f.....g idea @@ -85,9 +89,10 @@ fn json_from_string(content: &str, hint: FormatHint) -> Result try_loaders(content, &[Json, Yaml, Toml, Graphql]), - FormatHint::Yaml => try_loaders(content, &[Yaml, Json, Toml, Xml, Graphql]), - FormatHint::Toml => try_loaders(content, &[Toml, Json, Yaml, Xml, Graphql]), - FormatHint::Xml => try_loaders(content, &[Xml, Json, Yaml, Toml, Graphql]), + FormatHint::Yaml => try_loaders(content, &[Yaml, Json]), + FormatHint::Toml => try_loaders(content, &[Toml]), + FormatHint::Xml => try_loaders(content, &[Xml]), + FormatHint::Csv => try_loaders(content, &[Csv]), FormatHint::Graphql => try_loaders(content, &[Graphql, Json, Yaml, Toml, Xml]), } } @@ -98,6 +103,7 @@ fn try_loaders(content: &str, formats: &[FormatHint]) -> Result = None; let mut toml_error: Option<::toml::de::Error> = None; let mut xml_error: Option<::minidom::Error> = None; + let mut csv_error: Option<::csv::Error> = None; let mut graphql_error: Option = None; for hint in formats { @@ -126,6 +132,12 @@ fn try_loaders(content: &str, formats: &[FormatHint]) -> Result e, }); } + FormatHint::Csv => { + csv_error = Some(match csv::CsvLoader::json_from_str(content) { + Ok(json) => return Ok(json), + Err(e) => e, + }); + } FormatHint::Graphql => { graphql_error = Some(match graphql::GraphqlLoader::json_from_str(content) { Ok(json) => return Ok(json), @@ -144,6 +156,7 @@ fn try_loaders(content: &str, formats: &[FormatHint]) -> Result Result<(), LoaderError> { + let _result = DocumentPath::parse("./_samples/username.csv")?.load_raw()?; + dbg!(_result); + Ok(()) + } + #[allow(clippy::result_large_err)] #[test] #[ignore] diff --git a/codegenr/src/loaders/xml.rs b/codegenr/src/loaders/xml.rs index cab971f..0a812b0 100644 --- a/codegenr/src/loaders/xml.rs +++ b/codegenr/src/loaders/xml.rs @@ -3,7 +3,7 @@ use quickxml_to_serde::xml_str_to_json; pub struct XmlLoader {} impl DocumentLoader for XmlLoader { - type Error = minidom::Error; + type Error = minidom::error::Error; fn json_from_str(content: &str) -> Result { let config = quickxml_to_serde::Config::default(); xml_str_to_json(content, &config) diff --git a/codegenr/src/processor/file.rs b/codegenr/src/processor/file.rs index 1c51f6d..3c43801 100644 --- a/codegenr/src/processor/file.rs +++ b/codegenr/src/processor/file.rs @@ -20,7 +20,7 @@ impl Instruction for FileInstruction { } fn start(&self, params: Vec) -> Result, ProcessorError> { let file_path = params - .get(0) + .first() .ok_or(ProcessorError::InstructionParameterMissing(FILE, "file_name"))?; Ok(Box::new(FileLineHandler::new(&self.output_folder, file_path)?) as Box) }