diff --git a/.vscode/settings.json b/.vscode/settings.json index 89f67fd..d883179 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,7 @@ "source.fixAll.sortJSON": false // "source.organizeImports": true }, + "evenBetterToml.formatter.alignComments": true, // "rust-analyzer.checkOnSave": false, // "editor.formatOnSave": false "files.exclude": { diff --git a/Cargo.lock b/Cargo.lock index 64e15b5..dfbc8ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,10 +120,52 @@ dependencies = [ ] [[package]] -name = "arc-swap" -version = "0.4.8" +name = "async-channel" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-fs" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279cf904654eeebfa37ac9bb1598880884924aab82e290aa65c9e77a0e142e06" +dependencies = [ + "async-lock", + "autocfg", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-task" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" + +[[package]] +name = "async-walkdir" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826d88d73e87e7504b635b6e427561faa6a65f4a2f59e75efcbfa51a0876bb90" +dependencies = [ + "async-fs", + "futures-lite", +] [[package]] name = "atk" @@ -149,6 +191,12 @@ dependencies = [ "system-deps 6.1.1", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" @@ -209,6 +257,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c36a4d0d48574b3dd360b4b7d95cc651d2b6557b6402848a27d4b228a473e2a" +dependencies = [ + "async-channel", + "async-lock", + "async-task", + "fastrand 2.0.1", + "futures-io", + "futures-lite", + "piper", + "tracing", +] + [[package]] name = "brotli" version = "3.3.4" @@ -373,10 +437,8 @@ checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-targets", ] @@ -500,16 +562,19 @@ dependencies = [ ] [[package]] -name = "convert_case" -version = "0.4.0" +name = "concurrent-queue" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +dependencies = [ + "crossbeam-utils", +] [[package]] name = "convert_case" -version = "0.5.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a24b1aaf0fd0ce8b45161144d6f42cd91677fd5940fd431183eb023b3a2b8" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation" @@ -580,6 +645,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", + "futures", "is-terminal", "itertools", "num-traits", @@ -592,6 +658,7 @@ dependencies = [ "serde_derive", "serde_json", "tinytemplate", + "tokio", "walkdir", ] @@ -728,7 +795,9 @@ dependencies = [ "dar2oar_core", "log", "pretty_assertions", - "simple-log", + "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -736,6 +805,7 @@ name = "dar2oar_core" version = "0.1.6" dependencies = [ "anyhow", + "async-walkdir", "criterion", "jwalk", "log", @@ -743,9 +813,11 @@ dependencies = [ "pretty_assertions", "serde", "serde_json", - "simple-log", "thiserror", - "walkdir", + "tokio", + "tokio-stream", + "tracing", + "tracing-subscriber", ] [[package]] @@ -798,7 +870,7 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ - "convert_case 0.4.0", + "convert_case", "proc-macro2", "quote", "rustc_version", @@ -930,6 +1002,21 @@ dependencies = [ "libc", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -1026,6 +1113,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -1033,6 +1135,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1058,6 +1161,21 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.28" @@ -1069,6 +1187,12 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + [[package]] name = "futures-task" version = "0.3.28" @@ -1081,9 +1205,13 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -1105,11 +1233,13 @@ dependencies = [ "anyhow", "dar2oar_core", "log", + "once_cell", "pretty_assertions", - "simple-log", "tauri", "tauri-build", "tauri-plugin-log", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1499,15 +1629,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "iana-time-zone" version = "0.1.57" @@ -1638,12 +1759,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "is_debug" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d198e9919d9822d5f7083ba8530e04de87841eaf21ead9af8f2304efd57c89" - [[package]] name = "itertools" version = "0.10.5" @@ -1785,12 +1900,6 @@ dependencies = [ "safemem", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.4.7" @@ -1813,40 +1922,9 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" dependencies = [ - "serde", "value-bag", ] -[[package]] -name = "log-mdc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" - -[[package]] -name = "log4rs" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e1ad45e4584824d760c35d71868dd7e6e5acd8f5195a9573743b369fc86cd6" -dependencies = [ - "arc-swap", - "chrono", - "flate2", - "fnv", - "humantime", - "libc", - "log", - "log-mdc", - "parking_lot 0.11.2", - "serde", - "serde-value", - "serde_derive", - "serde_json", - "serde_yaml", - "thread-id", - "winapi", -] - [[package]] name = "loom" version = "0.5.6" @@ -2151,15 +2229,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "ordered-float" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" -dependencies = [ - "num-traits", -] - [[package]] name = "os_pipe" version = "1.1.4" @@ -2202,15 +2271,10 @@ dependencies = [ ] [[package]] -name = "parking_lot" -version = "0.11.2" +name = "parking" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -2219,21 +2283,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -2381,6 +2431,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -2513,12 +2574,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quick-xml" version = "0.29.0" @@ -2644,12 +2699,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.16" @@ -2852,16 +2901,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-value" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a65a7291a8a568adcae4c10a677ebcedbc6c9cec91c054dee2ce40b0e3290eb" -dependencies = [ - "ordered-float", - "serde", -] - [[package]] name = "serde_derive" version = "1.0.188" @@ -2933,18 +2972,6 @@ dependencies = [ "syn 2.0.37", ] -[[package]] -name = "serde_yaml" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" -dependencies = [ - "indexmap 1.9.3", - "ryu", - "serde", - "yaml-rust", -] - [[package]] name = "serialize-to-javascript" version = "0.1.1" @@ -3013,20 +3040,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" -[[package]] -name = "simple-log" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110feefe2a808cadb7fe8c07c615ac82e899f1795fec063ed2fb1d72de5d417b" -dependencies = [ - "convert_case 0.5.0", - "is_debug", - "log", - "log4rs", - "once_cell", - "serde", -] - [[package]] name = "siphasher" version = "0.3.11" @@ -3099,7 +3112,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "phf_shared 0.10.0", "precomputed-hash", "serde", @@ -3205,7 +3218,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "png", "raw-window-handle", "scopeguard", @@ -3458,7 +3471,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.0.1", "redox_syscall 0.3.5", "rustix", "windows-sys 0.48.0", @@ -3501,17 +3514,6 @@ dependencies = [ "syn 2.0.37", ] -[[package]] -name = "thread-id" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" -dependencies = [ - "libc", - "redox_syscall 0.1.57", - "winapi", -] - [[package]] name = "thread_local" version = "1.1.7" @@ -3579,14 +3581,37 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", "num_cpus", "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", ] [[package]] @@ -3634,11 +3659,10 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3646,9 +3670,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -3657,9 +3681,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -3825,6 +3849,12 @@ dependencies = [ "libc", ] +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + [[package]] name = "walkdir" version = "2.4.0" @@ -4377,15 +4407,6 @@ dependencies = [ "libc", ] -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yansi" version = "0.5.1" diff --git a/dar2oar_cli/Cargo.toml b/dar2oar_cli/Cargo.toml index fa397f5..bb8d3de 100644 --- a/dar2oar_cli/Cargo.toml +++ b/dar2oar_cli/Cargo.toml @@ -16,10 +16,12 @@ path = "./src/main.rs" [dependencies] anyhow = { version = "1.0.75", features = ["backtrace"] } -clap = { version = "4.4.4", features = ["derive"] } # For CLI +clap = { version = "4.4.4", features = ["derive"] } # For CLI dar2oar_core = { path = "../dar2oar_core" } -log = "0.4.20" # Logger -simple-log = "1.6.0" # Logger +log = "0.4.20" # Logger +tokio = { version = "1.33.0", features = ["fs", "rt", "macros"] } +tracing = "0.1.40" +tracing-subscriber = "0.3.17" [dev-dependencies] pretty_assertions = "1.4.0" diff --git a/dar2oar_cli/src/lib.rs b/dar2oar_cli/src/lib.rs index aebe9f0..13f1f05 100644 --- a/dar2oar_cli/src/lib.rs +++ b/dar2oar_cli/src/lib.rs @@ -4,7 +4,9 @@ use dar2oar_core::{ fs::{parallel, ConvertOptions}, read_mapping_table, }; -use std::path::PathBuf; +use std::fs::File; +use std::{path::PathBuf, str::FromStr}; +use tracing::Level; /// dar2oar --src "DAR path" --dist "OAR path" #[derive(Debug, Parser)] @@ -42,54 +44,38 @@ pub struct Args { hide_dar: bool, } -pub fn run_cli(args: Args) -> anyhow::Result<()> { - let config = simple_log::LogConfigBuilder::builder() - .path(args.log_path) - .size(100) - .roll_count(10) - .level(args.log_level) - .output_file() - .output_console() - .build(); - simple_log::new(config).unwrap(); +pub async fn run_cli(args: Args) -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(File::create(&args.log_path)?) + .with_max_level(Level::from_str(&args.log_level).unwrap_or(Level::ERROR)) + .init(); - let dist: Option = args.dist.map(|dist| PathBuf::from(&dist)); + macro_rules! read_table { + ($path:expr) => { + match $path { + Some(table_path) => { + let mapping = read_mapping_table(table_path).await?; + Some(mapping) + } + None => None, + } + }; + } - let table = match args.mapping_file { - Some(table_path) => { - let mapping = read_mapping_table(table_path)?; - Some(mapping) - } - None => None, - }; - - let table_1person = match args.mapping_1person_file { - Some(table_path) => { - let mapping = read_mapping_table(table_path)?; - Some(mapping) - } - None => None, + let config = ConvertOptions { + dar_dir: args.src, + oar_dir: args.dist.map(|dist| PathBuf::from(&dist)), + mod_name: args.name.as_deref(), + author: args.author.as_deref(), + section_table: read_table!(args.mapping_file), + section_1person_table: read_table!(args.mapping_1person_file), + hide_dar: args.hide_dar, }; let msg = match args.run_parallel { - true => parallel::convert_dar_to_oar(ConvertOptions { - dar_dir: args.src, - oar_dir: dist, - mod_name: args.name.as_deref(), - author: args.author.as_deref(), - section_table: table, - section_1person_table: table_1person, - hide_dar: args.hide_dar, - }), - false => convert_dar_to_oar(ConvertOptions { - dar_dir: args.src, - oar_dir: dist, - mod_name: args.name.as_deref(), - author: args.author.as_deref(), - section_table: table, - section_1person_table: table_1person, - hide_dar: args.hide_dar, - }), + true => parallel::convert_dar_to_oar(config).await, + false => convert_dar_to_oar(config).await, }?; log::debug!("{}", msg); Ok(()) diff --git a/dar2oar_cli/src/main.rs b/dar2oar_cli/src/main.rs index 9f7fcdd..591e6d3 100644 --- a/dar2oar_cli/src/main.rs +++ b/dar2oar_cli/src/main.rs @@ -1,6 +1,7 @@ use clap::Parser as _; use dar2oar_cli::{run_cli, Args}; -fn main() -> anyhow::Result<()> { - run_cli(Args::parse()) +#[tokio::main] +async fn main() -> anyhow::Result<()> { + run_cli(Args::parse()).await } diff --git a/dar2oar_core/Cargo.toml b/dar2oar_core/Cargo.toml index afa56ed..4712cfb 100644 --- a/dar2oar_core/Cargo.toml +++ b/dar2oar_core/Cargo.toml @@ -12,18 +12,30 @@ rust-version = "1.60" [dependencies] anyhow = { version = "1.0.75", features = ["backtrace"] } -log = "0.4.20" # Logger -nom = { version = "7.1.3", features = ["alloc"] } # Syntax -serde = { version = "1.0", features = ["derive"] } # Implement (De)Serializer -serde_json = "1.0" # Json converter -thiserror = "1.0.48" # define errors type -jwalk = "0.8.1" # To parallel traverse dir recursivly -walkdir = "2.4.0" # To traverse dir recursivly(Single thiread) +async-walkdir = "0.2.0" # To traverse dir recursivly(Single thiread) +jwalk = "0.8.1" # To parallel traverse dir recursivly +log = "0.4.20" # Logger +nom = { version = "7.1.3", features = ["alloc"] } # Syntax +serde = { version = "1.0", features = ["derive"] } # Implement (De)Serializer +serde_json = "1.0" # Json converter +thiserror = "1.0.48" # define errors type +tokio = { version = "1.33.0", features = [ + "fs", + "io-util", + "macros", + "rt", +] } # Async Executor +tokio-stream = "0.1.14" # Async next() method +tracing = "0.1.40" [dev-dependencies] -criterion = { version = "0.5.1", features = ["html_reports"] } +criterion = { version = "0.5.1", features = [ + "async_futures", + "async_tokio", + "html_reports", +] } pretty_assertions = "1.4.0" -simple-log = "1.6.0" +tracing-subscriber = "0.3.17" [[bench]] diff --git a/dar2oar_core/benches/convert_n_thread.rs b/dar2oar_core/benches/convert_n_thread.rs index ffda5c0..3d4de01 100644 --- a/dar2oar_core/benches/convert_n_thread.rs +++ b/dar2oar_core/benches/convert_n_thread.rs @@ -1,3 +1,4 @@ +use criterion::async_executor::FuturesExecutor; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use dar2oar_core::{ convert_dar_to_oar, @@ -5,54 +6,46 @@ use dar2oar_core::{ read_mapping_table, }; use std::time::Duration; +use tokio::fs; const REMOVE_TARGET: &str = "../test/data/UNDERDOG Animations/meshes/actors/character/animations/OpenAnimationReplacer"; const TARGET: &str = "../test/data/UNDERDOG Animations"; +const TABLE_PATH: &str = "../test/settings/UnderDog Animations_v1.9.6_mapping_table.txt"; fn criterion_benchmark(c: &mut Criterion) { - let config = simple_log::LogConfigBuilder::builder() - .path("../convert.log") - .size(100) - .roll_count(10) - .level("error") - .output_file() - .output_console() - .build(); - simple_log::new(config).unwrap(); - let mut group = c.benchmark_group("dar2oar sequential vs parallel"); group.warm_up_time(Duration::from_secs(70)).sample_size(10); group.bench_function("dar2oar multi thread", |b| { - b.iter(|| { + b.to_async(FuturesExecutor).iter(|| async { if std::path::Path::new(REMOVE_TARGET).exists() { - std::fs::remove_dir_all(REMOVE_TARGET).unwrap(); + fs::remove_dir_all(REMOVE_TARGET).await.unwrap(); } - let table_content = "../test/settings/mapping_table.txt"; - let mapping = read_mapping_table(table_content).unwrap(); + let mapping = read_mapping_table(TABLE_PATH).await.unwrap(); parallel::convert_dar_to_oar(black_box(ConvertOptions { dar_dir: TARGET, section_table: Some(mapping), ..Default::default() })) + .await }) }); group.bench_function("dar2oar single thread", |b| { - b.iter(|| { + b.to_async(FuturesExecutor).iter(|| async { if std::path::Path::new(REMOVE_TARGET).exists() { - std::fs::remove_dir_all(REMOVE_TARGET).unwrap(); + fs::remove_dir_all(REMOVE_TARGET).await.unwrap(); } - let table_content = "../test/settings/mapping_table.txt"; - let mapping = read_mapping_table(table_content).unwrap(); + let mapping = read_mapping_table(TABLE_PATH).await.unwrap(); convert_dar_to_oar(black_box(ConvertOptions { dar_dir: TARGET, section_table: Some(mapping), ..Default::default() })) + .await }) }); diff --git a/dar2oar_core/src/fs/mapping_table.rs b/dar2oar_core/src/fs/mapping_table.rs index 10b4ff3..43183a8 100644 --- a/dar2oar_core/src/fs/mapping_table.rs +++ b/dar2oar_core/src/fs/mapping_table.rs @@ -1,13 +1,14 @@ use anyhow::bail; use std::collections::HashMap; -use std::fs::File; -use std::io::Read; use std::path::Path; +use tokio::{fs::File, io::AsyncReadExt}; -pub fn read_mapping_table(table_path: impl AsRef) -> anyhow::Result> { +pub async fn read_mapping_table( + table_path: impl AsRef, +) -> anyhow::Result> { let mut file_contents = String::new(); - match File::open(table_path) { - Ok(mut file) => match file.read_to_string(&mut file_contents) { + match File::open(table_path).await { + Ok(mut file) => match file.read_to_string(&mut file_contents).await { Ok(_) => Ok(parse_mapping_table(&file_contents)), Err(e) => bail!("Error reading file: {}", e), }, @@ -19,7 +20,7 @@ fn parse_mapping_table(table: &str) -> HashMap { let mut map = HashMap::new(); // Sequential numbering of duplicate keys when no key is available. - let mut current_section_name = String::new(); + let mut current_section_name = ""; let mut idx = 0; for line in table.lines() { if line.starts_with("//") { @@ -27,24 +28,27 @@ fn parse_mapping_table(table: &str) -> HashMap { }; let mapping = match line.find(' ') { - Some(idx) => line.split_at(idx), - None => (line, ""), + Some(idx) => { + let (key, val) = line.split_at(idx); + (key, Some(val)) + } + None => (line, None), }; let section_name = match mapping.1 { - "" => { + None => { idx += 1; format!("{}_{}", current_section_name, idx) } - val => { - current_section_name = val.trim().to_string(); + Some(val) => { + current_section_name = val.trim(); idx = 0; - current_section_name.clone() + current_section_name.to_string() } }; match mapping.0 { "" | "\r\n" | "\n" => continue, // Skip blank lines. - key => map.insert(key.to_string(), section_name.clone()), + key => map.insert(key.into(), section_name), }; } @@ -76,13 +80,13 @@ mod tests { let result = parse_mapping_table(input); let mut expected = HashMap::new(); - expected.insert("8000000".to_string(), "Combat".to_string()); - expected.insert("8000001".to_string(), "Base".to_string()); - expected.insert("8000002".to_string(), "Base_1".to_string()); - expected.insert("8000005".to_string(), "Female".to_string()); - expected.insert("8001000".to_string(), "Unarmed".to_string()); - expected.insert("8001010".to_string(), "Sword".to_string()); - expected.insert("8001020".to_string(), "Sword+Shield".to_string()); + expected.insert("8000000".into(), "Combat".to_string()); + expected.insert("8000001".into(), "Base".to_string()); + expected.insert("8000002".into(), "Base_1".to_string()); + expected.insert("8000005".into(), "Female".to_string()); + expected.insert("8001000".into(), "Unarmed".to_string()); + expected.insert("8001010".into(), "Sword".to_string()); + expected.insert("8001020".into(), "Sword+Shield".to_string()); assert_eq!(result, expected); } diff --git a/dar2oar_core/src/fs/mod.rs b/dar2oar_core/src/fs/mod.rs index 1ec9099..547e1fb 100644 --- a/dar2oar_core/src/fs/mod.rs +++ b/dar2oar_core/src/fs/mod.rs @@ -1,17 +1,21 @@ mod mapping_table; -pub mod parallel; -pub mod path_changer; mod sequential; -pub use mapping_table::read_mapping_table; -pub use sequential::convert_dar_to_oar; +pub mod parallel; +pub mod path_changer; use crate::conditions::{ConditionsConfig, MainConfig}; use anyhow::Context as _; +use async_walkdir::WalkDir; use std::collections::HashMap; -use std::fs; -use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; +use tokio::fs; +use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; +use tokio_stream::StreamExt; +use tracing::trace; + +pub use mapping_table::read_mapping_table; +pub use sequential::convert_dar_to_oar; #[derive(Debug, Default, PartialEq)] pub struct ConvertOptions<'a, P: AsRef> { @@ -31,35 +35,35 @@ pub struct ConvertOptions<'a, P: AsRef> { pub hide_dar: bool, } -fn read_file

(file_path: P) -> io::Result +async fn read_file

(file_path: P) -> io::Result where P: AsRef, { - let mut file = fs::File::open(file_path)?; + let mut file = fs::File::open(file_path).await?; let mut content = String::new(); - file.read_to_string(&mut content)?; + file.read_to_string(&mut content).await?; Ok(content) } -fn write_section_config

(oar_dir: P, config_json: ConditionsConfig) -> anyhow::Result<()> +async fn write_section_config

(oar_dir: P, config_json: ConditionsConfig) -> anyhow::Result<()> where P: AsRef, { let target_path = oar_dir.as_ref().join("config.json"); - let mut config_file = fs::File::create(&target_path).with_context(|| { + let mut config_file = fs::File::create(&target_path).await.with_context(|| { let msg = format!("writing section config target: {:?}", target_path); - log::error!("{}", msg); + tracing::error!("{}", msg); msg })?; let json = serde_json::to_string_pretty(&config_json)?; - config_file.write_all(json.as_bytes())?; + config_file.write_all(json.as_bytes()).await?; Ok(()) } /// If there is no name_space_config file, create one. /// If it exists, do nothing. (This behavior is intended to facilitate the creation of config files /// for 1st_person and 3rd_person.) -fn write_name_space_config

( +async fn write_name_space_config

( oar_name_space_path: P, mod_name: &str, author: Option<&str>, @@ -77,10 +81,10 @@ where author: author.unwrap_or_default().into(), ..Default::default() }; - fs::create_dir_all(&oar_name_space_path)?; - let mut config_file = fs::File::create(target_file)?; + fs::create_dir_all(&oar_name_space_path).await?; + let mut config_file = fs::File::create(target_file).await?; let json = serde_json::to_string_pretty(&config_json)?; - config_file.write_all(json.as_bytes())?; + config_file.write_all(json.as_bytes()).await?; Ok(()) } @@ -89,12 +93,13 @@ where /// /// # NOTE /// It is currently used only in GUI, but is implemented in Core as an API. -pub fn restore_dar(dar_dir: impl AsRef) -> anyhow::Result { +pub async fn restore_dar(dar_dir: impl AsRef) -> anyhow::Result { let mut restored_dar = None; let mut restored_1st_dar = None; - for entry in walkdir::WalkDir::new(dar_dir) { - let entry = entry?; - let path = entry.path(); + let mut entries = WalkDir::new(dar_dir); + while let Some(entry) = entries.next().await { + let path = entry?.path(); + let path = path.as_path(); let (dar_root, _, is_1st_person, _, _, _) = match path_changer::parse_dar_path(path, Some("DynamicAnimationReplacer.mohidden")) { Ok(data) => data, @@ -116,7 +121,7 @@ pub fn restore_dar(dar_dir: impl AsRef) -> anyhow::Result { .as_os_str() .to_string_lossy() .replace(".mohidden", ""); - fs::rename(dar_root.clone(), dist)?; + fs::rename(dar_root.clone(), dist).await?; msg = format!("{}- Restored 3rd_person", msg); } if let Some(dar_root) = restored_1st_dar.as_ref() { @@ -124,7 +129,7 @@ pub fn restore_dar(dar_dir: impl AsRef) -> anyhow::Result { .as_os_str() .to_string_lossy() .replace(".mohidden", ""); - fs::rename(dar_root.clone(), dist)?; + fs::rename(dar_root.clone(), dist).await?; msg = format!("{}\n- Restored 1rd_person", msg); } @@ -137,12 +142,13 @@ pub fn restore_dar(dar_dir: impl AsRef) -> anyhow::Result { /// # NOTE /// It is currently used only in GUI, but is implemented in Core as an API. -pub fn remove_oar(dar_dir: impl AsRef) -> anyhow::Result<()> { +pub async fn remove_oar(dar_dir: impl AsRef) -> anyhow::Result<()> { let mut restored_dar = None; let mut restored_1st_dar = None; - for entry in walkdir::WalkDir::new(dar_dir) { - let entry = entry?; - let path = entry.path(); + let mut entries = WalkDir::new(dar_dir); + while let Some(entry) = entries.next().await { + let path = entry?.path(); + let path = path.as_path(); // NOTE: The OAR root obtained by parse fn is calculated and not guaranteed to exist. let (dar_root, oar_name_space_path, is_1st_person, _, _, _) = match path_changer::parse_dar_path(path, Some("DynamicAnimationReplacer.mohidden")) { @@ -165,23 +171,16 @@ pub fn remove_oar(dar_dir: impl AsRef) -> anyhow::Result<()> { } if let Some((_, oar_root)) = restored_dar { - dbg!(&oar_root); + trace!("{:?}", &oar_root); if oar_root.exists() { - fs::remove_dir_all(oar_root)?; + fs::remove_dir_all(oar_root).await?; } } if let Some((_, oar_root)) = restored_1st_dar { - dbg!(&oar_root); + trace!("{:?}", &oar_root); if oar_root.exists() { - fs::remove_dir_all(oar_root)?; + fs::remove_dir_all(oar_root).await?; } } Ok(()) } - -#[ignore] -#[test] -fn should_restore_dar() { - let dar_dir = "../test/data/UNDERDOG Animations"; - remove_oar(dar_dir).unwrap(); -} diff --git a/dar2oar_core/src/fs/parallel.rs b/dar2oar_core/src/fs/parallel.rs index 2d9dfa0..74a5a1b 100644 --- a/dar2oar_core/src/fs/parallel.rs +++ b/dar2oar_core/src/fs/parallel.rs @@ -4,13 +4,13 @@ use crate::fs::path_changer::parse_dar_path; use crate::fs::{read_file, write_name_space_config, write_section_config, ConvertOptions}; use anyhow::{bail, Context as _, Result}; use jwalk::WalkDir; -use std::fs; use std::path::Path; +use tokio::fs; /// multi thread converter /// # Return /// Complete info -pub fn convert_dar_to_oar(options: ConvertOptions>) -> Result { +pub async fn convert_dar_to_oar(options: ConvertOptions<'_, impl AsRef>) -> Result { let ConvertOptions { dar_dir, oar_dir, @@ -73,9 +73,9 @@ pub fn convert_dar_to_oar(options: ConvertOptions>) -> Result { log::trace!("Content:\n{}", content); @@ -86,7 +86,7 @@ pub fn convert_dar_to_oar(options: ConvertOptions>) -> Result log::error!("Error reading file {path:?}: {err}"), } @@ -101,6 +101,7 @@ pub fn convert_dar_to_oar(options: ConvertOptions>) -> Result>) -> Result>) -> Result>) -> Result { +pub async fn convert_dar_to_oar(options: ConvertOptions<'_, impl AsRef>) -> Result { let ConvertOptions { dar_dir, oar_dir, @@ -25,15 +26,24 @@ pub fn convert_dar_to_oar(options: ConvertOptions>) -> Result