diff --git a/Cargo.lock b/Cargo.lock index 9070d17..aa7fcac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,27 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "alsa" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fe60779335388a88c01ac6c3be40304d1e349de3ada3b15f7808bb90fa9dce" +dependencies = [ + "alsa-sys", + "bitflags 2.6.0", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.5.2" @@ -179,12 +200,109 @@ dependencies = [ "libc", ] +[[package]] +name = "annotate-snippets" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e" +dependencies = [ + "unicode-width", + "yansi-term", +] + +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "apple-bindgen-helmer-fork" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9717d1bd9d5fb13ef1de642273bac67e654664b69e99a65b7738cc9cbf0ca5e4" +dependencies = [ + "apple-sdk", + "bindgen", + "clap", + "derive_more", + "regex", + "serde", + "thiserror", + "toml 0.6.0", +] + +[[package]] +name = "apple-sdk" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a04f192a700686ee70008ff4e4eb76fe7d11814ab93b7ee9d48c36b9a9f0bd2a" +dependencies = [ + "plist", + "serde", + "serde_json", +] + +[[package]] +name = "apple-sys-helmer-fork" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03b84426ef7620d5554496dd85d4f80ab3d348cbc929d4ec8ea0ed6ddf6b075" +dependencies = [ + "apple-bindgen-helmer-fork", + "apple-sdk", + "objc", +] + [[package]] name = "approx" version = "0.5.1" @@ -609,6 +727,30 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "annotate-snippets", + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.69", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -816,6 +958,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-expr" version = "0.15.8" @@ -853,6 +1004,69 @@ dependencies = [ "libc", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.4", +] + +[[package]] +name = "clap" +version = "4.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.69", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + +[[package]] +name = "clipboard-rs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93c547618248086cfc855d71ccfbf6cc8f64b7866ebb8616e313dedcfc33b1e2" +dependencies = [ + "clipboard-win", + "cocoa", + "image 0.25.1", + "x11rb", +] + [[package]] name = "clipboard-win" version = "5.4.0" @@ -860,6 +1074,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" dependencies = [ "error-code", + "windows-win", +] + +[[package]] +name = "cocoa" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6140449f97a6e97f9511815c5632d84c8aacf8ac271ad77c559218161a1373c" +dependencies = [ + "bitflags 1.3.2", + "block", + "cocoa-foundation", + "core-foundation", + "core-graphics", + "foreign-types", + "libc", + "objc", +] + +[[package]] +name = "cocoa-foundation" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6234cbb2e4c785b456c0644748b1ac416dd045799740356f8363dfe00c93f7" +dependencies = [ + "bitflags 1.3.2", + "block", + "core-foundation", + "core-graphics-types", + "libc", + "objc", ] [[package]] @@ -878,6 +1123,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + [[package]] name = "com" version = "0.6.0" @@ -928,6 +1179,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -957,6 +1232,19 @@ dependencies = [ "libc", ] +[[package]] +name = "core-graphics-helmer-fork" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32eb7c354ae9f6d437a6039099ce7ecd049337a8109b23d73e48e8ffba8e9cd5" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + [[package]] name = "core-graphics-types" version = "0.1.3" @@ -968,6 +1256,49 @@ dependencies = [ "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f01585027057ff5f0a5bf276174ae4c1594a2c5bde93d5f46a016d76270f5a9" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -1033,12 +1364,38 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-url" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" +[[package]] +name = "dbus" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bb21987b9fb1613058ba3843121dd18b163b254d8a6e797e144cbac14d96d1b" +dependencies = [ + "libc", + "libdbus-sys", + "winapi", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1050,6 +1407,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +dependencies = [ + "convert_case 0.4.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.69", +] + [[package]] name = "digest" version = "0.10.7" @@ -1499,6 +1869,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -1506,6 +1891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1514,6 +1900,17 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -1577,6 +1974,7 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1648,6 +2046,12 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "glow" version = "0.13.1" @@ -1765,7 +2169,7 @@ checksum = "9c08c1f623a8d0b722b8b99f821eb0ba672a1618f0d3b16ddbee1cedd2dd8557" dependencies = [ "bitflags 2.6.0", "gpu-descriptor-types", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -1787,6 +2191,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1851,6 +2261,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "icrate" version = "0.0.4" @@ -1953,6 +2369,16 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1960,7 +2386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -1994,6 +2420,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itertools" version = "0.12.1" @@ -2003,6 +2435,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + [[package]] name = "jni" version = "0.21.1" @@ -2078,6 +2516,18 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lebe" version = "0.5.2" @@ -2090,6 +2540,16 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libdbus-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06085512b750d640299b79be4bad3d2fa90a9c00b1fd9e1b46364f66f0485c72" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "libfuzzer-sys" version = "0.4.7" @@ -2138,6 +2598,34 @@ dependencies = [ "redox_syscall 0.4.1", ] +[[package]] +name = "libspa" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65f3a4b81b2a2d8c7f300643676202debd1b7c929dbf5c9bb89402ea11d19810" +dependencies = [ + "bitflags 2.6.0", + "cc", + "convert_case 0.6.0", + "cookie-factory", + "libc", + "libspa-sys", + "nix 0.27.1", + "nom", + "system-deps", +] + +[[package]] +name = "libspa-sys" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0d9716420364790e85cbb9d3ac2c950bde16a7dd36f3209b7dfdfc4a24d01f" +dependencies = [ + "bindgen", + "cc", + "system-deps", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2181,6 +2669,15 @@ dependencies = [ "imgref", ] +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -2312,7 +2809,7 @@ dependencies = [ "bitflags 2.6.0", "codespan-reporting", "hexf-parse", - "indexmap", + "indexmap 2.2.6", "log", "num-traits", "rustc-hash", @@ -2386,6 +2883,17 @@ dependencies = [ "memoffset 0.7.1", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.29.0" @@ -2415,12 +2923,30 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "noop_proc_macro" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -2454,6 +2980,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.4.2" @@ -2544,6 +3076,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" dependencies = [ "malloc_buf", + "objc_exception", +] + +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", ] [[package]] @@ -2687,6 +3231,24 @@ dependencies = [ "objc2-metal", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + [[package]] name = "object" version = "0.36.1" @@ -2696,6 +3258,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -2800,12 +3385,53 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pipewire" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08e645ba5c45109106d56610b3ee60eb13a6f2beb8b74f8dc8186cf261788dda" +dependencies = [ + "anyhow", + "bitflags 2.6.0", + "libc", + "libspa", + "libspa-sys", + "nix 0.27.1", + "once_cell", + "pipewire-sys", + "thiserror", +] + +[[package]] +name = "pipewire-sys" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "849e188f90b1dda88fe2bfe1ad31fe5f158af2c98f80fb5d13726c44f3f01112" +dependencies = [ + "bindgen", + "libspa-sys", + "system-deps", +] + [[package]] name = "pkg-config" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "plist" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42cf17e9a1800f5f396bc67d193dc9411b59012a5876445ef450d449881e1016" +dependencies = [ + "base64 0.22.1", + "indexmap 2.2.6", + "quick-xml 0.32.0", + "serde", + "time", +] + [[package]] name = "png" version = "0.17.13" @@ -2850,6 +3476,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2862,6 +3494,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.69", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -2924,6 +3566,24 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3a6e5838b60e0e8fa7a43f22ade549a37d61f8bdbe636d0d7816191de969c2" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.34.0" @@ -3194,6 +3854,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.27" @@ -3253,6 +3922,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + [[package]] name = "safe_arch" version = "0.7.2" @@ -3271,6 +3946,30 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scap" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8759a122294835ac0df383a04e60b41834bc911edc4b93329bfcfd1fe0ab70fa" +dependencies = [ + "apple-sys-helmer-fork", + "bytes", + "core-graphics-helmer-fork", + "cpal", + "dbus", + "hound", + "image 0.24.9", + "itertools", + "pipewire", + "rand", + "screencapturekit", + "screencapturekit-sys", + "sysinfo", + "tao-core-video-sys", + "windows 0.52.0", + "windows-capture", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3283,6 +3982,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "screencapturekit" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5eeeb57ac94960cfe5ff4c402be6585ae4c8d29a2cf41b276048c2e849d64e" +dependencies = [ + "screencapturekit-sys", +] + +[[package]] +name = "screencapturekit-sys" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22411b57f7d49e7fe08025198813ee6fd65e1ee5eff4ebc7880c12c82bde4c60" +dependencies = [ + "block", + "dispatch", + "objc", + "objc-foundation", + "objc_id", + "once_cell", +] + [[package]] name = "sctk-adwaita" version = "0.8.3" @@ -3296,6 +4018,12 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.204" @@ -3316,6 +4044,17 @@ dependencies = [ "syn 2.0.69", ] +[[package]] +name = "serde_json" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_repr" version = "0.1.19" @@ -3347,6 +4086,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -3521,6 +4266,12 @@ dependencies = [ "float-cmp", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -3542,6 +4293,7 @@ name = "swiftmouse" version = "0.1.0" dependencies = [ "ashpd", + "clipboard-rs", "edge-detection", "eframe", "egui_extras", @@ -3549,7 +4301,10 @@ dependencies = [ "imageproc", "once_cell", "rayon", + "scap", "tokio", + "xcap", + "zbus 4.3.1", ] [[package]] @@ -3574,6 +4329,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "windows 0.52.0", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -3583,10 +4353,22 @@ dependencies = [ "cfg-expr", "heck", "pkg-config", - "toml", + "toml 0.8.14", "version-compare", ] +[[package]] +name = "tao-core-video-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271450eb289cb4d8d0720c6ce70c72c8c858c93dd61fc625881616752e6b98f6" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "objc", +] + [[package]] name = "target-lexicon" version = "0.12.14" @@ -3645,6 +4427,37 @@ dependencies = [ "weezl", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-skia" version = "0.11.4" @@ -3716,6 +4529,18 @@ dependencies = [ "syn 2.0.69", ] +[[package]] +name = "toml" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.5.1", + "toml_edit 0.18.1", +] + [[package]] name = "toml" version = "0.8.14" @@ -3724,10 +4549,19 @@ checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.6", "toml_edit 0.22.14", ] +[[package]] +name = "toml_datetime" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.6.6" @@ -3737,14 +4571,27 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +dependencies = [ + "indexmap 1.9.3", + "nom8", + "serde", + "serde_spanned", + "toml_datetime 0.5.1", +] + [[package]] name = "toml_edit" version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap", - "toml_datetime", + "indexmap 2.2.6", + "toml_datetime 0.6.6", "winnow 0.5.40", ] @@ -3754,8 +4601,8 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap", - "toml_datetime", + "indexmap 2.2.6", + "toml_datetime 0.6.6", "winnow 0.5.40", ] @@ -3765,10 +4612,10 @@ version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ - "indexmap", + "indexmap 2.2.6", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.6", "winnow 0.6.13", ] @@ -3961,6 +4808,12 @@ dependencies = [ "tiny-skia-path", ] +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "v_frame" version = "0.3.8" @@ -4165,7 +5018,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edf466fc49a4feb65a511ca403fec3601494d0dee85dbf37fff6fa0dd4eec3b6" dependencies = [ "proc-macro2", - "quick-xml", + "quick-xml 0.34.0", "quote", ] @@ -4271,7 +5124,7 @@ dependencies = [ "cfg_aliases 0.1.1", "codespan-reporting", "document-features", - "indexmap", + "indexmap 2.2.6", "log", "naga", "once_cell", @@ -4338,6 +5191,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.34", +] + [[package]] name = "wide" version = "0.7.25" @@ -4391,8 +5256,8 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.48.0", + "windows-interface 0.48.0", "windows-targets 0.48.5", ] @@ -4402,10 +5267,42 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132" +dependencies = [ + "windows-core 0.56.0", "windows-targets 0.52.6", ] +[[package]] +name = "windows-capture" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e2f94b43842205eb84814f505badade792e7b8cd132e436636ccbf7baa08fc" +dependencies = [ + "parking_lot", + "rayon", + "thiserror", + "windows 0.56.0", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -4415,6 +5312,28 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6" +dependencies = [ + "windows-implement 0.56.0", + "windows-interface 0.56.0", + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-implement" version = "0.48.0" @@ -4426,6 +5345,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "windows-implement" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.69", +] + [[package]] name = "windows-interface" version = "0.48.0" @@ -4437,6 +5367,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "windows-interface" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.69", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -4510,6 +5460,15 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-win" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e23e33622b3b52f948049acbec9bcc34bf6e26d74176b88941f213c75cf2dc" +dependencies = [ + "error-code", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -4741,6 +5700,35 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d" +[[package]] +name = "xcap" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a462fc23e8aab59b6dbd7d717999d4589374ef94142005143e40740187847ccb" +dependencies = [ + "core-foundation", + "core-graphics", + "dbus", + "image 0.25.1", + "log", + "percent-encoding", + "sysinfo", + "thiserror", + "windows 0.54.0", + "xcb", +] + +[[package]] +name = "xcb" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e75181b5a62b6eeaa72f303d3cef7dbb841e22885bf6d3e66fe23e88c55dc6" +dependencies = [ + "bitflags 1.3.2", + "libc", + "quick-xml 0.30.0", +] + [[package]] name = "xcursor" version = "0.3.5" @@ -4788,6 +5776,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" +[[package]] +name = "yansi-term" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" +dependencies = [ + "winapi", +] + [[package]] name = "zbus" version = "3.15.2" diff --git a/Cargo.toml b/Cargo.toml index b790c0e..1c905fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] ashpd = { git = "https://github.com/bilelmoussaoui/ashpd", version = "0.9.0" } +clipboard-rs = "0.1.7" edge-detection = "0.2.6" eframe = { version = "0.28.1", features = [ "default", @@ -15,4 +16,19 @@ image = "0.25.1" imageproc = "0.25.0" once_cell = "1.19.0" rayon = "1.10.0" +scap = "0.0.5" tokio = { version = "1.38.0", features = ["full"] } +xcap = "0.0.10" +zbus = "4.3.1" + +[lib] +name = "swiftmouse" +path = "src/lib.rs" + +[[bin]] +name = "daemon" +path = "src/bin/daemon.rs" + +[[bin]] +name = "gui" +path = "src/bin/gui.rs" diff --git a/src/autotype.rs b/src/autotype/mod.rs similarity index 100% rename from src/autotype.rs rename to src/autotype/mod.rs diff --git a/src/bin/daemon.rs b/src/bin/daemon.rs new file mode 100644 index 0000000..761b7f6 --- /dev/null +++ b/src/bin/daemon.rs @@ -0,0 +1,506 @@ +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] +#![allow(rustdoc::missing_crate_level_docs)] + +use core::num; +use std::cmp; +use std::process::Stdio; + +use ashpd::desktop::print; +use image::GenericImage; +use image::GenericImageView; +use rayon::iter::IndexedParallelIterator; +use rayon::iter::IntoParallelRefIterator; +use rayon::iter::ParallelIterator; +use rayon::vec; +use swiftmouse::bounding_box; +use swiftmouse::image_utils; +use swiftmouse::screenshot; +use swiftmouse::globalshortcut; +use zbus::zvariant::Endian; +use zbus::zvariant::WriteBytes; + +mod gui; + +const SCREENSHOT_PATH: &str = "/tmp/screenshot.png"; + +fn get_pixel_gray(image: &image::ImageBuffer, Vec>, mut x: i16, mut y: i16) -> u16 { + if x < 0 { + x = 0; + } + if y < 0 { + y = 0; + } + if x >= image.width() as i16 { + x = image.width() as i16 - 1; + } + if y >= image.height() as i16 { + y = image.height() as i16 - 1; + } + + let pixel = image.get_pixel(x as u32, y as u32); + return pixel.0[0] as u16 + pixel.0[1] as u16 + pixel.0[2] as u16; +} + +#[tokio::main] +async fn main() { + let (mut rx, _conn) = globalshortcut::listen().await; + println!("[Main] Waiting for events"); + let mut screenshot_tool = screenshot::get_screenshot_tool(); + let screenshot = screenshot_tool.take_screenshot().await.unwrap(); + + + while let Some(_) = rx.recv().await { + let total_start = std::time::Instant::now(); + let screenshot: image::ImageBuffer, Vec> = screenshot_tool.take_screenshot().await.unwrap(); + // let dynamic_image = image::DynamicImage::ImageRgb8(screenshot.clone()); + // write to /tmp/screenshot.png + screenshot.save("/tmp/screenshot.png").unwrap(); + let start = std::time::Instant::now(); + + //gray image + // let gray_image = dynamic_image.grayscale(); + // println!("Gray Elapsed: {:?}", start.elapsed()); + // 3x3 edge conv + let start = std::time::Instant::now(); + let downsampled_map = vec![vec![false; (screenshot.width() / 2) as usize]; (screenshot.width() / 2) as usize]; + let downsampled_map = downsampled_map.par_iter().enumerate().map(|(x, row)| { + let mut line = vec![false; row.len()]; + for y in 0..row.len() { + let mut field = vec![vec![0; 4]; 4]; + for i in -1..3 { + for j in -1..3 { + field[(i+1) as usize][(j+1) as usize] = get_pixel_gray(&screenshot, (x as i16)*2 + i, (y as i16)*2 + j); + } + } + + let mut tl_edge: i16 = 0; + for i in 0..3 { + for j in 0..3 { + if i == 1 && j == 1 { + tl_edge += field[i][j] as i16 * 8; + } else { + tl_edge -= field[i][j] as i16; + } + } + } + if tl_edge > 0 { + line[y] = true; + continue; + } + + + let mut tr_edge: i16 = 0; + for i in 0..3 { + for j in 0..3 { + if i == 1 && j == 1 { + tr_edge += field[i+1][j] as i16 * 8; + } else { + tr_edge -= field[i+1][j] as i16; + } + } + } + if tr_edge > 0 { + line[y] = true; + continue; + } + + let mut bl_edge: i16 = 0; + for i in 0..3 { + for j in 0..3 { + if i == 1 && j == 1 { + bl_edge += field[i][j+1] as i16 * 8; + } else { + bl_edge -= field[i][j+1] as i16; + } + } + } + if bl_edge > 0 { + line[y] = true; + continue; + } + + let mut br_edge: i16 = 0; + for i in 0..3 { + for j in 0..3 { + if i == 1 && j == 1 { + br_edge += field[i+1][j+1] as i16 * 8; + } else { + br_edge -= field[i+1][j+1] as i16; + } + } + } + if br_edge > 0 { + line[y] = true; + continue; + } + } + line + }).collect::>>(); + println!("Downsampled Elapsed: {:?}", start.elapsed()); + + // to debug img + // let mut downsampled_image = image::DynamicImage::new_rgb8(screenshot.width()/2, screenshot.height()/2); + // for x in 0..downsampled_image.width() { + // for y in 0..downsampled_image.height() { + // if downsampled_map[x as usize][y as usize] { + // downsampled_image.put_pixel(x, y, image::Rgba([255, 255, 255, 255])); + // } else { + // downsampled_image.put_pixel(x, y, image::Rgba([0, 0, 0, 255])); + // } + // } + // } + // // save downsampled image + // downsampled_image.save("/tmp/downsampled.png").unwrap(); + + + // let start = std::time::Instant::now(); + // let mut edge_image = gray_image.filter3x3(&[ + // -1.0, -1.0, -1.0, + // -1.0, 8.0, -1.0, + // -1.0, -1.0, -1.0, + // ]); + // println!("Edge Elapsed: {:?}", start.elapsed()); + // save edge image + // edge_image.save("/tmp/edge.png").unwrap(); + // if > 0 then 1 + // for x in 0..edge_image.width() { + // for y in 0..edge_image.height() { + // let pixel = edge_image.get_pixel(x, y); + // if pixel[0] > 0 { + // edge_image.put_pixel(x, y, image::Rgba([255, 255, 255, 255])); + // } else { + // edge_image.put_pixel(x, y, image::Rgba([0, 0, 0, 255])); + // } + // } + // } + // save edge image + // edge_image.save("/tmp/edge2.png").unwrap(); + // downsample to half size manually + // let start = std::time::Instant::now(); + // let downsampled_image_map = downsampled_map; + // let mut downsampled_image_map = vec![vec![false; (edge_image.width() / 2) as usize]; (edge_image.width() / 2) as usize]; + // let mut downsampled_image1 = image::DynamicImage::new_rgb8(edge_image.width()/2, edge_image.height()/2); + // for x in 0..edge_image.width()/2 { + // for y in 0..edge_image.height()/2 { + // // if one pixel is > 0 + // if ( + // edge_image.get_pixel(x*2, y*2)[0] > 0 || + // edge_image.get_pixel(x*2+1, y*2)[0] > 0 || + // edge_image.get_pixel(x*2, y*2+1)[0] > 0 || + // edge_image.get_pixel(x*2+1, y*2+1)[0] > 0 + // ) { + // downsampled_image_map[x as usize][y as usize] = true; + // downsampled_image1.put_pixel(x, y, image::Rgba([255, 255, 255, 255])); + // } else { + // downsampled_image_map[x as usize][y as usize] = false; + // downsampled_image1.put_pixel(x, y, image::Rgba([0, 0, 0, 255])); + // } + // } + // } + // println!("Downsampled Elapsed: {:?}", start.elapsed()); + // // save downsampled image + // downsampled_image1.save("/tmp/downsampled1.png").unwrap(); + // map to true fals + // for x in 0..downsampled_image.width() { + // for y in 0..downsampled_image.height() { + // let pixel = downsampled_image.get_pixel(x, y); + // if pixel[0] > 0 { + // downsampled_image_map[x as usize][y as usize] = true; + // } + // } + // } + let downsampled_image_map = downsampled_map; + + let start = std::time::Instant::now(); + let num_threads = rayon::current_num_threads(); + // println!("Num threads: {:?}", num_threads); + let width = downsampled_image_map.len(); + let num_chunks = num_threads; + let chunk_size = width / num_chunks; + let mut chunks: Vec<(usize, usize)> = Vec::new(); + loop { + let start = chunks.len() * chunk_size; + let end = start + chunk_size; + if end >= width { + chunks.push((start, width-1)); + break; + } + chunks.push((start, end)); + } + println!("Chunks: {:?}", chunks); + let boxes = chunks.par_iter().enumerate().map(|(i, (start, end))| { + let width = downsampled_image_map.len(); + let height = downsampled_image_map[0].len(); + let mut visited = vec![vec![false; height]; width as usize]; + let mut boxes = Vec::new(); + for x in *start..*end { + for y in 0..height { + if downsampled_image_map[x][y] && !visited[x][y] { + let mut start_x = x; + let mut end_x = x; + let mut start_y = y; + let mut end_y = y; + + let mut queue = std::collections::VecDeque::new(); + queue.push_back((x, y)); + visited[x][y] = true; + while let Some((x, y)) = queue.pop_front() { + if x < start_x { + start_x = x; + } + if x > end_x { + end_x = x; + } + if y < start_y { + start_y = y; + } + if y > end_y { + end_y = y; + } + + if x > 0 && downsampled_image_map[x - 1][y] && !visited[x - 1][y] { + queue.push_back((x-1, y)); + visited[x - 1][y] = true; + } + if x < width - 1 && downsampled_image_map[x + 1][y] && !visited[x + 1][y] { + queue.push_back((x+1, y)); + visited[x + 1][y] = true; + } + if y > 0 && downsampled_image_map[x][y - 1] && !visited[x][y - 1] { + queue.push_back((x, y-1)); + visited[x][y - 1] = true; + } + if y < height - 1 && downsampled_image_map[x][y + 1] && !visited[x][y + 1] { + queue.push_back((x, y+1)); + visited[x][y + 1] = true; + } + } + + // println!("Chunk {:?} Box: {:?} {:?} {:?} {:?}", i, start_x, start_y, end_x, end_y); + boxes.push((cmp::max(start_x as i32 -1, 0) as usize, + cmp::max(start_y as i32 -1, 0) as usize, + cmp::min(end_x as i32 +1, width as i32 -1) as usize, + cmp::min(end_y as i32 +1, height as i32 -1) as usize)); + } + } + } + boxes + }).collect::>>().concat(); + println!("Boxes Elapsed: {:?}", start.elapsed()); + + let start = std::time::Instant::now(); + let big_boxes = boxes.clone().into_iter().filter(|(min_x, min_y, max_x, max_y)| { + (max_y - min_y) > 15 + }).collect::>(); + println!("Big box Elapsed: {:?}", start.elapsed()); + + let start = std::time::Instant::now(); + let large_images = boxes.clone().into_iter().filter(|(min_x, min_y, max_x, max_y)| { + (max_y - min_y) > 300 && (max_x - min_x) > 300 && *min_x > 10 && *max_x < width - 10 + }).collect::>(); + println!("large img Elapsed: {:?}", start.elapsed()); + + let start = std::time::Instant::now(); + let text_boxes = boxes.clone().into_iter().filter(|(min_x, min_y, max_x, max_y)| { + (max_y - min_y) <= 15 + }).filter(|(min_x, min_y, max_x, max_y)| { + let mut in_video = false; + for video in &large_images { + if *min_x >= video.0 && *min_y >= video.1 && *max_x <= video.2 && *max_y <= video.3 { + in_video = true; + break; + } + } + !in_video + }).collect::>(); + println!("Video filter Elapsed: {:?}", start.elapsed()); + + let start = std::time::Instant::now(); + // create lines by merging text boxes that are close on x and aligned on 1 + let mut lines: Vec> = Vec::new(); + let mut text_boxes: Vec<(usize, usize, usize, usize)> = text_boxes.clone(); + let mut handled: Vec = vec![-1; text_boxes.len()]; + for i in 0..text_boxes.len() { + if handled[i] == -1 { + let mut line = Vec::new(); + line.push(text_boxes[i]); + lines.push(line); + handled[i] = lines.len() as i32 - 1; + } + + for j in 0..text_boxes.len() { + if handled[j] != -1 { + continue; + } + let (min_x, min_y, max_x, max_y) = text_boxes[j]; + let (min_x1, min_y1, max_x1, max_y1) = text_boxes[i]; + if (min_y1 as i32 - min_y as i32).abs() <= 3 && ((max_x1 as i32 - min_x as i32).abs() <= 4 || (max_x as i32 - min_x1 as i32).abs() <= 4) { + lines[handled[i] as usize].push(text_boxes[j]); + handled[j] = handled[i]; + } + } + } + + // create bounding box per line + let mut line_boxes = Vec::new(); + for line in lines { + let mut min_x = line[0].0; + let mut min_y = line[0].1; + let mut max_x = line[0].2; + let mut max_y = line[0].3; + for (x, y, x1, y1) in line { + if x < min_x { + min_x = x; + } + if y < min_y { + min_y = y; + } + if x1 > max_x { + max_x = x1; + } + if y1 > max_y { + max_y = y1; + } + } + line_boxes.push((min_x, min_y, max_x, max_y)); + } + + let start = std::time::Instant::now(); + // small images are big boxes that are > 50% white + let mut small_images = Vec::new(); + for (min_x, min_y, max_x, max_y) in big_boxes.clone() { + let mut white = 0; + let mut total = 0; + for x in min_x..max_x { + for y in min_y..max_y { + if x >= downsampled_image_map.len() as usize || y >= downsampled_image_map[0].len() as usize { + continue; + } + total += 1; + if downsampled_image_map[x][y] { + white += 1; + } + } + } + if white as f32 / total as f32 > 0.5 { + small_images.push((min_x, min_y, max_x, max_y)); + } + } + println!("Small img Elapsed: {:?}", start.elapsed()); + + + println!("Num boxes: {:?}", boxes.len()); + println!("Num small boxes: {:?}", text_boxes.len()); + println!("Num big boxes: {:?}", big_boxes.len()); + println!("Num ultra large boxes: {:?}", large_images.len()); + println!("Num lines: {:?}", line_boxes.len()); + println!("Num small images: {:?}", small_images.len()); + println!("Total Elapsed: {:?}", total_start.elapsed()); + + + // image_utils::draw_boxes(&mut downsampled_image, &text_boxes, image::Rgba([255, 0, 0, 255])); + // image_utils::draw_boxes(&mut downsampled_image, &big_boxes, image::Rgba([0, 255, 0, 255])); + // image_utils::draw_boxes(&mut downsampled_image, &large_images, image::Rgba([0, 255, 255, 255])); + // image_utils::draw_boxes(&mut downsampled_image, &line_boxes, image::Rgba([255, 255, 0, 255])); + // image_utils::draw_boxes(&mut downsampled_image, &small_images, image::Rgba([255, 0, 255, 255])); + // save downsampled image + // downsampled_image.save("/tmp/downsampled2.png").unwrap(); + + let mut binpath = std::env::current_exe().unwrap(); + binpath.set_file_name("gui"); + let gui_binpath = binpath.to_str().unwrap(); + println!("[Main] GUI binpath: {:?}", gui_binpath); + + let mut child = std::process::Command::new(gui_binpath) + .stdin(Stdio::piped()) + .spawn() + .unwrap(); + match child.stdin.as_mut() { + Some(stdin) => { + write_boxes(stdin, &unmap_downsampled_boxes(&big_boxes)); + write_boxes(stdin, &unmap_downsampled_boxes(&line_boxes)); + write_boxes(stdin, &unmap_downsampled_boxes(&small_images)); + write_boxes(stdin, &unmap_downsampled_boxes(&large_images)); + } + None => { + println!("[Main] Failed to open stdin"); + } + } + // wait for proc exit + child.wait().unwrap(); + } + + println!("[Main] Exiting"); +} + +fn unmap_downsampled_boxes(boxes: &Vec<(usize, usize, usize, usize)>) -> Vec<(usize, usize, usize, usize)> { + let mut new_boxes = Vec::new(); + for (min_x, min_y, max_x, max_y) in boxes { + new_boxes.push(( + *min_x * 2, + *min_y * 2, + *max_x * 2, + *max_y * 2 + )); + } + new_boxes +} + +fn write_boxes(stdin: &mut std::process::ChildStdin, boxes: &Vec<(usize, usize, usize, usize)>) { + let boxes_len = boxes.len() as u32; + stdin.write_u32(Endian::Little, boxes_len).unwrap(); + for box_ in boxes { + stdin.write_u32(Endian::Little, box_.0 as u32).unwrap(); + stdin.write_u32(Endian::Little, box_.1 as u32).unwrap(); + stdin.write_u32(Endian::Little, box_.2 as u32).unwrap(); + stdin.write_u32(Endian::Little, box_.3 as u32).unwrap(); + } +} + +async fn run(screenshot: &image::ImageBuffer, Vec>) { + println!("[Main] Opening screenshot"); + let start_time = std::time::Instant::now(); + screenshot.save(SCREENSHOT_PATH).unwrap(); + println!("[Main] Elapsed: {:?}", start_time.elapsed()); + + println!("[Main] Finding bounding boxes"); + let start_time = std::time::Instant::now(); + let dynamic_image = image::DynamicImage::ImageRgb8(screenshot.clone()); + let (text_boxes, big_boxes)/* bounding_boxes */ = bounding_box::find_bounding_boxes_v2(&dynamic_image); + println!("[Main] Elapsed: {:?}", start_time.elapsed()); + println!("[Main] Found {:?} small boxes and {:?} big boxes", text_boxes.len(), big_boxes.len()); + + // std::process::exit(0); + + // binpath + let mut binpath = std::env::current_exe().unwrap(); + binpath.set_file_name("gui"); + let gui_binpath = binpath.to_str().unwrap(); + println!("[Main] GUI binpath: {:?}", gui_binpath); + + let mut child = std::process::Command::new(gui_binpath) + .stdin(Stdio::piped()) + .spawn() + .unwrap(); + match child.stdin.as_mut() { + Some(stdin) => { + // write len of big boxes + let big_boxes_len = big_boxes.len() as u32; + stdin.write_u32(Endian::Little, big_boxes_len).unwrap(); + for big_box in big_boxes { + // write big boxes + stdin.write_u32(Endian::Little, big_box.0).unwrap(); + stdin.write_u32(Endian::Little, big_box.1).unwrap(); + stdin.write_u32(Endian::Little, big_box.2).unwrap(); + stdin.write_u32(Endian::Little, big_box.3).unwrap(); + } + } + None => { + println!("[Main] Failed to open stdin"); + } + } + // wait for proc exit + child.wait().unwrap(); + + println!("[Main] GUI closed"); +} \ No newline at end of file diff --git a/src/bin/gui.rs b/src/bin/gui.rs new file mode 100644 index 0000000..027567d --- /dev/null +++ b/src/bin/gui.rs @@ -0,0 +1,32 @@ +use zbus::zvariant::Endian; +use zbus::zvariant::ReadBytes; + +const SCREENSHOT_PATH: &str = "/tmp/screenshot.png"; + +fn read_boxes(stdin: &mut std::io::Stdin) -> Vec<(u32, u32, u32, u32)> { + let boxes_len = stdin.read_u32(Endian::Little).unwrap(); + let mut boxes = Vec::new(); + for _ in 0..boxes_len { + let x = stdin.read_u32(Endian::Little).unwrap(); + let y = stdin.read_u32(Endian::Little).unwrap(); + let width = stdin.read_u32(Endian::Little).unwrap(); + let height = stdin.read_u32(Endian::Little).unwrap(); + boxes.push((x, y, width, height)); + } + boxes +} + +#[tokio::main] +pub async fn main() { + let mut stdin = std::io::stdin(); + let big_boxes = read_boxes(&mut stdin); + let line_boxes = read_boxes(&mut stdin); + let small_images = read_boxes(&mut stdin); + let large_images = read_boxes(&mut stdin); + + // start autoclick session + swiftmouse::autotype::start_autoclick_session().await.unwrap(); + // screen width and height + swiftmouse::gui::show_gui( big_boxes, line_boxes, small_images, large_images, SCREENSHOT_PATH.to_string()); + +} \ No newline at end of file diff --git a/src/bounding_box.rs b/src/bounding_box.rs index 5616668..949e593 100644 --- a/src/bounding_box.rs +++ b/src/bounding_box.rs @@ -12,47 +12,74 @@ enum Direction { } pub fn find_horizontal_lines(image: &DynamicImage) -> Vec> { - let mut horizontal_lines = vec![Vec::new(); image.height() as usize]; - for y in 0..image.height() { - let mut current_color = image.get_pixel(0, y); - let mut start_x = 0; - for x in 1..image.width() { - let color = image.get_pixel(x, y); - if color != current_color || x == image.width()-1 { - // if length is greater than 10 - if x - start_x > 40 { - horizontal_lines[y as usize].push((start_x, x)); + // with rayon + let mapped_lines: Vec> = (0..image.height()).collect::>() + .par_iter() + .map(|index| { + let mut lines = Vec::new(); + let mut current_color = image.get_pixel(0, *index as u32); + let mut start_x = 0; + for x in 1..image.width() { + let color = image.get_pixel(x, *index as u32); + if color != current_color || x == image.width()-1 { + // if length is greater than 10 + if x - start_x > 40 { + lines.push((start_x, x)); + } + + start_x = x; + current_color = color; } - - start_x = x; - current_color = color; } - } - } + lines + }) + .collect(); - return horizontal_lines; + return mapped_lines; } pub fn find_vertical_lines(image: &DynamicImage) -> Vec> { - let mut vertical_lines = vec![Vec::new(); image.width() as usize]; - for x in 0..image.width() { - let mut current_color = image.get_pixel(x, 0); - let mut start_y = 0; - for y in 1..image.height() { - let color = image.get_pixel(x, y); - if color != current_color || y == image.height()-1 { - // if length is greater than 10 - if y - start_y > 20 { - vertical_lines[x as usize].push((start_y, y)); + // let mut vertical_lines = vec![Vec::new(); image.width() as usize]; + // for x in 0..image.width() { + // let mut current_color = image.get_pixel(x, 0); + // let mut start_y = 0; + // for y in 1..image.height() { + // let color = image.get_pixel(x, y); + // if color != current_color || y == image.height()-1 { + // // if length is greater than 10 + // if y - start_y > 20 { + // vertical_lines[x as usize].push((start_y, y)); + // } + // + // start_y = y; + // current_color = color; + // } + // } + // } + + let mapped_lines = (0..image.width()).collect::>() + .par_iter() + .map(|index| { + let mut lines = Vec::new(); + let mut current_color = image.get_pixel(*index as u32, 0); + let mut start_y = 0; + for y in 1..image.height() { + let color = image.get_pixel(*index as u32, y); + if color != current_color || y == image.height()-1 { + // if length is greater than 10 + if y - start_y > 20 { + lines.push((start_y, y)); + } + + start_y = y; + current_color = color; } - - start_y = y; - current_color = color; } - } - } + lines + }) + .collect(); - return vertical_lines; + return mapped_lines; } fn draw_lines(image: &DynamicImage, lines: &Vec>, direction: Direction, color: image::Rgba) -> DynamicImage { @@ -264,7 +291,7 @@ enum EdgeType { Before, After, None, Both } -fn check_edge(image: &DynamicImage, lines: &Vec>, direction: Direction) -> Vec> { +fn detect_edge_types(image: &DynamicImage, lines: &Vec>, direction: Direction) -> Vec> { let lines_with_index = lines.iter().enumerate().collect::>(); let mapped_lines: Vec> = lines_with_index .par_iter() @@ -426,9 +453,9 @@ pub fn find_bounding_boxes_v2(image: &DynamicImage) -> (Vec<(u32, u32, u32, u32) } println!("Elapsed: {:?}", start_time.elapsed()); - println!("Checking edges"); + println!("Detect horizontal edge types"); let start_time = std::time::Instant::now(); - let horizontal_edges = check_edge(image, &horizontal_lines, Direction::Horizontal); + let horizontal_edges = detect_edge_types(image, &horizontal_lines, Direction::Horizontal); if write_debug_images { // filter to only before lines let before_lines = horizontal_edges.iter().map(|x| x.iter().filter(|(_, _, edge_type)| *edge_type == EdgeType::Before).map(|(start, end, _)| (start.clone(), end.clone())).collect::>()).collect::>(); @@ -442,8 +469,8 @@ pub fn find_bounding_boxes_v2(image: &DynamicImage) -> (Vec<(u32, u32, u32, u32) } println!("Elapsed: {:?}", start_time.elapsed()); - println!("Checking vertical edges"); - let vertical_edges = check_edge(image, &vertical_lines, Direction::Vertical); + println!("Detect vertical edge types"); + let vertical_edges = detect_edge_types(image, &vertical_lines, Direction::Vertical); if write_debug_images { // filter to only before lines let before_lines = vertical_edges.iter().map(|x| x.iter().filter(|(_, _, edge_type)| *edge_type == EdgeType::Before).map(|(start, end, _)| (start.clone(), end.clone())).collect::>()).collect::>(); @@ -745,7 +772,7 @@ pub fn find_bounding_boxes_v2(image: &DynamicImage) -> (Vec<(u32, u32, u32, u32) let mut small_boxes = Vec::new(); for (start_x, start_y, end_x, end_y) in filtered_boxes.iter() { let max_textfragment_height = 35; - let max_textfragment_height_2 = 25; + let max_textfragment_height_2 = 35; if end_y - start_y > max_textfragment_height { big_boxes.push((start_x.clone(), start_y.clone(), end_x.clone(), end_y.clone())); } else if end_y - start_y > max_textfragment_height_2 { @@ -941,7 +968,7 @@ pub fn find_bounding_boxes_v2(image: &DynamicImage) -> (Vec<(u32, u32, u32, u32) // debug_img.save("/tmp/swiftmouse_0_no_children_boxes.png").unwrap(); // drop all big boxes with endx bigger than 70 - let big_boxes = big_boxes.iter().filter(|(_, _, end_x, _)| *end_x < 70).map(|(start_x, start_y, end_x, end_y)| (start_x.clone(), start_y.clone(), end_x.clone(), end_y.clone())).collect::>(); + // let big_boxes = big_boxes.iter().filter(|(_, _, end_x, _)| *end_x < 70).map(|(start_x, start_y, end_x, end_y)| (start_x.clone(), start_y.clone(), end_x.clone(), end_y.clone())).collect::>(); if write_debug_images { let mut debug_img = image.clone(); @@ -950,9 +977,9 @@ pub fn find_bounding_boxes_v2(image: &DynamicImage) -> (Vec<(u32, u32, u32, u32) draw_horizontal_line_colored(&mut debug_img, *end_y, *start_x, *end_x, image::Rgba([255, 255, 0, 255])); draw_vertical_line_colored(&mut debug_img, *start_x, *start_y, *end_y, image::Rgba([255, 0, 255, 255])); draw_vertical_line_colored(&mut debug_img, *end_x, *start_y, *end_y, image::Rgba([0, 255, 255, 255])); - // draw greenline at offset - let offset = start_y + end_y % 1000; - draw_vertical_line_colored(&mut debug_img, *start_x + offset, *start_y, *end_y, image::Rgba([0, 255, 0, 255])); + // // draw greenline at offset + // let offset = start_y + end_y % 1000; + // draw_vertical_line_colored(&mut debug_img, *start_x + offset, *start_y, *end_y, image::Rgba([0, 255, 0, 255])); } debug_img.save("/tmp/swiftmouse_0_big_boxes_filtered.png").unwrap(); } @@ -1077,11 +1104,6 @@ pub fn find_bounding_boxes_v2(image: &DynamicImage) -> (Vec<(u32, u32, u32, u32) return (smallboxes, filtered_big_boxes); } -fn overlapping_area(start_x_1: &u32, start_y_1: &u32, end_x_1: &u32, end_y_1: &u32, start_x_2: &u32, start_y_2: &u32, end_x_2: &u32, end_y_2: &u32) -> u32 { - // Max(0, Min(XA2, XB2) - Max(XA1, XB1)) * Max(0, Min(YA2, YB2) - Max(YA1, YB1)) - cmp::max(0, cmp::min(*end_x_1, *end_x_2) as i32 - cmp::max(*start_x_1, *start_x_2) as i32) as u32 * cmp::max(0, cmp::min(*end_y_1, *end_y_2) as i32 - cmp::max(*start_y_1, *start_y_2) as i32) as u32 -} - fn draw_horizontal_line_colored(image: &mut DynamicImage, y: u32, start_x: u32, end_x: u32, color: image::Rgba) { for x in start_x..end_x { if x >= image.width() { @@ -1098,144 +1120,4 @@ fn draw_vertical_line_colored(image: &mut DynamicImage, x: u32, start_y: u32, en } image.put_pixel(x, y, color); } -} - -pub fn merge_overlapping_bounding_boxes(bounding_boxes: Vec<(u32, u32, u32, u32)>) -> Vec<(u32, u32, u32, u32)> { - let mut merged_bounding_boxes: Vec<(u32, u32, u32, u32)> = Vec::new(); - let mut visited_list: Vec = vec![false; bounding_boxes.len()]; - let mut did_merge = false; - - for (i, bounding_box) in bounding_boxes.iter().enumerate(){ - if visited_list[i] { - continue; - } - visited_list[i] = true; - let mut current_bb = *bounding_box; - - for (j, bounding_box2) in bounding_boxes.iter().enumerate(){ - if visited_list[j] { - continue; - } - if bounds_overlap(current_bb, *bounding_box2) { - visited_list[j] = true; - current_bb = merge_bounds(current_bb, *bounding_box2); - did_merge = true; - } - } - - // discard if width and height are less are more than 100 - if (current_bb.2 - current_bb.0) > 100 && (current_bb.3 - current_bb.1) > 100 { - continue; - } - - merged_bounding_boxes.push(current_bb); - } - - if did_merge { - return merge_overlapping_bounding_boxes(merged_bounding_boxes); - } else { - return merged_bounding_boxes; - } -} - -fn bounds_overlap(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32)) -> bool { - let (min_x, min_y, max_x, max_y) = a; - let (min_x2, min_y2, max_x2, max_y2) = b; - if min_x2 >= min_x && min_y2 >= min_y && max_x2 <= max_x && max_y2 <= max_y { - return true; - } - - if contains(a, (min_x2, min_y2)) || contains(a, (max_x2, max_y2)) || contains(a, (min_x2, max_y2)) || contains(a, (max_x2, min_y2)) { - return true; - } - - false -} - -fn contains(a: (u32, u32, u32, u32), b: (u32, u32)) -> bool { - let (min_x, min_y, max_x, max_y) = a; - let (x, y) = b; - if x >= min_x && x <= max_x && y >= min_y && y <= max_y { - return true; - } - false -} - -fn merge_bounds(a: (u32, u32, u32, u32), b: (u32, u32, u32, u32)) -> (u32, u32, u32, u32) { - let (min_x, min_y, max_x, max_y) = a; - let (min_x2, min_y2, max_x2, max_y2) = b; - let min_x = min_x.min(min_x2); - let min_y = min_y.min(min_y2); - let max_x = max_x.max(max_x2); - let max_y = max_y.max(max_y2); - (min_x, min_y, max_x, max_y) -} - -fn get_pixel_grayscale(image: &DynamicImage, x: u32, y: u32) -> i32 { - let [r,g,b,a] = image.get_pixel(x, y).0; - (r as u32 + g as u32 + b as u32) as i32 -} - -pub(crate) fn get_bounding_box_flood_fill(image: &DynamicImage, x: u32, y: u32, visited_bitmap: &mut Vec>) -> Option<(u32, u32, u32, u32)> { - let mut min_x = x; - let mut max_x = x; - let mut min_y = y; - let mut max_y = y; - - let mut open: Vec<(u32, u32)> = Vec::new(); - open.push((x, y)); - let mut pixel_count = 0; - let mut visited: Vec<(u32, u32)> = Vec::new(); - - while let Some((x, y)) = open.pop() { - if x >= image.width()-1 as u32 || y >= image.height()-1 as u32 || x <= 1 || y <= 1 { - continue; - } - - let edge_pixel = get_pixel_grayscale(image, x, y) * 4 - + -1 * get_pixel_grayscale(image, x, y+1) - + -1 * get_pixel_grayscale(image, x-1, y) - + -1 * get_pixel_grayscale(image, x+1, y) - + -1 * get_pixel_grayscale(image, x, y-1); - if edge_pixel < 100 { - continue; - } - - // check if pixel is already visited - if visited_bitmap[x as usize][y as usize] { - continue; - } - visited_bitmap[x as usize][y as usize] = true; - - if x < min_x { - min_x = x; - } - if x > max_x { - max_x = x; - } - if y < min_y { - min_y = y; - } - if y > max_y { - max_y = y; - } - - open.push((x+1, y)); - open.push((x-1, y)); - open.push((x, y+1)); - open.push((x, y-1)); - - pixel_count += 1; - visited.push((x, y)); - - if pixel_count > 500 { - return None - } - } - - if pixel_count < 5 { - return None - } - - Some((min_x, min_y, max_x, max_y)) } \ No newline at end of file diff --git a/src/globalshortcut/mod.rs b/src/globalshortcut/mod.rs new file mode 100644 index 0000000..9639855 --- /dev/null +++ b/src/globalshortcut/mod.rs @@ -0,0 +1,32 @@ +use tokio::sync::mpsc; +use zbus::{blocking::connection, interface}; + +struct ZbusListener { + // tx channel + tx: mpsc::Sender<()>, +} + +#[interface(name = "com.quexten.swiftmouse")] +impl ZbusListener { + async fn run(&mut self) -> String { + println!("sending"); + self.tx.send(()).await.unwrap(); + "".to_string() + } +} + + +// fn listen and have a return channel to send events +pub async fn listen() -> (mpsc::Receiver<()> , connection::Connection) { + let (tx, rx) = mpsc::channel(1); + let listener = ZbusListener { + tx, + }; + let _conn = connection::Builder::session().unwrap() + .name("com.quexten.swiftmouse").unwrap() + .serve_at("/com/quexten/swiftmouse", listener).unwrap() + .build() + .unwrap(); + println!("[Global Shortcuts] Listening on D-Bus"); + return (rx, _conn); +} \ No newline at end of file diff --git a/src/gui.rs b/src/gui.rs index fd8919d..5c43736 100644 --- a/src/gui.rs +++ b/src/gui.rs @@ -4,25 +4,24 @@ use eframe::egui::{self, InputState}; use crate::autotype::{self, ClickType}; -pub(crate) fn show_gui(positions: Vec<(u32, u32, u32, u32)>, big_boxes: Vec<(u32, u32, u32, u32)>, width: u32, height: u32, path: String) { +static COLOR_GRAY: egui::Color32 = egui::Color32::from_rgb(100, 100, 100); + +pub fn show_gui(big_boxes: Vec<(u32, u32, u32, u32)>, line_boxes: Vec<(u32, u32, u32, u32)>, small_images: Vec<(u32, u32, u32, u32)>, large_images: Vec<(u32, u32, u32, u32)>, path: String) { let mut options = eframe::NativeOptions { viewport: egui::ViewportBuilder::default().with_inner_size([400.0, 800.0]), ..Default::default() }; options.viewport.fullscreen = Some(true); - // options.viewport.maximized = Some(true); - let heap_width = Arc::new(width); - let heap_height = Arc::new(height); let _ = eframe::run_native( "Swiftmouse", options, Box::new(|cc| { egui_extras::install_image_loaders(&cc.egui_ctx); let mut app = Box::::default(); - app.positions = positions; + app.line_boxes = line_boxes; app.big_boxes = big_boxes; - app.width = Arc::try_unwrap(heap_width).unwrap(); - app.height = Arc::try_unwrap(heap_height).unwrap(); + app.small_images = small_images; + app.large_images = large_images; app.path = path; app.letters_typed = vec![]; Ok(app) @@ -32,66 +31,67 @@ pub(crate) fn show_gui(positions: Vec<(u32, u32, u32, u32)>, big_boxes: Vec<(u32 #[derive(Default)] struct MyApp { - positions: Vec<(u32, u32, u32, u32)>, + line_boxes: Vec<(u32, u32, u32, u32)>, big_boxes: Vec<(u32, u32, u32, u32)>, + small_images: Vec<(u32, u32, u32, u32)>, + large_images: Vec<(u32, u32, u32, u32)>, letters_typed: Vec, - width: u32, - height: u32, + selected_box: Option<(u32, u32, u32, u32)>, path: String, } fn get_key(i: &InputState) -> Option { - if i.key_pressed(egui::Key::A) { + if i.key_released(egui::Key::A) { return Some(0); - } else if i.key_pressed(egui::Key::B) { + } else if i.key_released(egui::Key::B) { return Some(1); - } else if i.key_pressed(egui::Key::C) { + } else if i.key_released(egui::Key::C) { return Some(2); - } else if i.key_pressed(egui::Key::D) { + } else if i.key_released(egui::Key::D) { return Some(3); - } else if i.key_pressed(egui::Key::E) { + } else if i.key_released(egui::Key::E) { return Some(4); - } else if i.key_pressed(egui::Key::F) { + } else if i.key_released(egui::Key::F) { return Some(5); - } else if i.key_pressed(egui::Key::G) { + } else if i.key_released(egui::Key::G) { return Some(6); - } else if i.key_pressed(egui::Key::H) { + } else if i.key_released(egui::Key::H) { return Some(7); - } else if i.key_pressed(egui::Key::I) { + } else if i.key_released(egui::Key::I) { return Some(8); - } else if i.key_pressed(egui::Key::J) { + } else if i.key_released(egui::Key::J) { return Some(9); - } else if i.key_pressed(egui::Key::K) { + } else if i.key_released(egui::Key::K) { return Some(10); - } else if i.key_pressed(egui::Key::L) { + } else if i.key_released(egui::Key::L) { return Some(11); - } else if i.key_pressed(egui::Key::M) { + } else if i.key_released(egui::Key::M) { return Some(12); - } else if i.key_pressed(egui::Key::N) { + } else if i.key_released(egui::Key::N) { return Some(13); - } else if i.key_pressed(egui::Key::O) { + } else if i.key_released(egui::Key::O) { return Some(14); - } else if i.key_pressed(egui::Key::P) { + } else if i.key_released(egui::Key::P) { return Some(15); - } else if i.key_pressed(egui::Key::Q) { + } else if i.key_released(egui::Key::Q) { return Some(16); - } else if i.key_pressed(egui::Key::R) { + } else if i.key_released(egui::Key::R) { return Some(17); - } else if i.key_pressed(egui::Key::S) { + } else if i.key_released(egui::Key::S) { return Some(18); - } else if i.key_pressed(egui::Key::T) { + } else if i.key_released(egui::Key::T) { return Some(19); - } else if i.key_pressed(egui::Key::U) { + } else if i.key_released(egui::Key::U) { return Some(20); - } else if i.key_pressed(egui::Key::V) { + } else if i.key_released(egui::Key::V) { return Some(21); - } else if i.key_pressed(egui::Key::W) { + } else if i.key_released(egui::Key::W) { return Some(22); - } else if i.key_pressed(egui::Key::X) { + } else if i.key_released(egui::Key::X) { return Some(23); - } else if i.key_pressed(egui::Key::Y) { + } else if i.key_released(egui::Key::Y) { return Some(24); - } else if i.key_pressed(egui::Key::Z) { + } else if i.key_released(egui::Key::Z) { return Some(25); } return None; @@ -109,40 +109,194 @@ fn get_index_for_letters(letter1: u8, letter2: u8) -> i32 { letter1 * 26 + letter2 } +impl MyApp { + fn draw_images(&mut self, ui: &mut egui::Ui) { + // if first letter is i + let is_selected = self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == 8; + let is_other_selected = self.letters_typed.len() > 0 && self.letters_typed[0] as u8 != 8; + if is_other_selected { + return + } + + let image_color = egui::Color32::from_rgb(255, 160, 50); + + for (index, (start_x, start_y, end_x, end_y)) in self.small_images.iter().enumerate() { + let (letter1, letter2) = get_letters_for_index(index as i32); + if self.letters_typed.len() > 1 { + if self.letters_typed[1] as u8 != letter1 { + continue + } + } + if self.letters_typed.len() > 2 { + if self.letters_typed[2] as u8 != letter2 { + continue + } else { + self.selected_box = Some((*start_x, *start_y, *end_x, *end_y)); + continue + } + } + if self.letters_typed.len() == 2 { + self.selected_box = None + } + + ui.painter().rect_stroke( + egui::Rect::from_min_max( + egui::pos2(*start_x as f32, *start_y as f32), + egui::pos2(*end_x as f32, *end_y as f32), + ), + 0.0, + egui::Stroke::new(2.0, image_color), + ); + // if is_selected { + let label = format!("{}{}", std::char::from_u32(letter1 as u32 + 65).unwrap(), std::char::from_u32(letter2 as u32 + 65).unwrap()); + ui.allocate_ui_at_rect(egui::Rect::from_min_max( + egui::pos2(*start_x as f32, *start_y as f32), + egui::pos2(*start_x as f32 + 100.0, *end_y as f32 + 100.0), + ), |ui| { + ui.label(egui::RichText::new(label).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(image_color)); + }); + // } + } + } + + fn draw_text_lines(&mut self, ui: &mut egui::Ui) { + let image_color = egui::Color32::from_rgb(150, 200, 20); + // let is_selected = self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == 11; + let is_other_selected = self.letters_typed.len() > 0 && self.letters_typed[0] as u8 != 11; + if is_other_selected { + return + } + + for (index, (start_x, start_y, end_x, end_y)) in self.line_boxes.iter().enumerate() { + let (letter1, letter2) = get_letters_for_index(index as i32); + if self.letters_typed.len() > 1 { + if self.letters_typed[1] as u8 != letter1 { + continue + } + } + if self.letters_typed.len() > 2 { + if self.letters_typed[2] as u8 != letter2 { + continue + } else { + self.selected_box = Some((*start_x, *start_y, *end_x, *end_y)); + continue + } + } + if self.letters_typed.len() == 2 { + self.selected_box = None + } + + ui.painter().rect_stroke( + egui::Rect::from_min_max( + egui::pos2(*start_x as f32, *start_y as f32), + egui::pos2(*end_x as f32, *end_y as f32), + ), + 0.0, + egui::Stroke::new(2.0, image_color), + ); + // if is_selected { + let label = format!("{}{}", std::char::from_u32(letter1 as u32 + 65).unwrap(), std::char::from_u32(letter2 as u32 + 65).unwrap()); + ui.allocate_ui_at_rect(egui::Rect::from_min_max( + egui::pos2(*start_x as f32, *start_y as f32), + egui::pos2(*start_x as f32 + 100.0, *end_y as f32 + 100.0), + ), |ui| { + ui.label(egui::RichText::new(label).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(image_color)); + }); + // } + } + } + + fn draw_big_boxes(&mut self, ui: &mut egui::Ui) { + let image_color = egui::Color32::from_rgb(150, 0, 150); + let is_selected = self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == 1; + let is_other_selected = self.letters_typed.len() > 0 && self.letters_typed[0] as u8 != 1; + if is_other_selected { + return + } + + for (index, (start_x, start_y, end_x, end_y)) in self.big_boxes.iter().enumerate() { + let (letter1, letter2) = get_letters_for_index(index as i32); + if self.letters_typed.len() > 1 { + if self.letters_typed[1] as u8 != letter1 { + continue + } + } + if self.letters_typed.len() > 2 { + if self.letters_typed[2] as u8 != letter2 { + continue + } else { + self.selected_box = Some((*start_x, *start_y, *end_x, *end_y)); + continue + } + } + if self.letters_typed.len() == 2 { + self.selected_box = None + } + + ui.painter().rect_stroke( + egui::Rect::from_min_max( + egui::pos2(*start_x as f32, *start_y as f32), + egui::pos2(*end_x as f32, *end_y as f32), + ), + 0.0, + egui::Stroke::new(2.0, image_color), + ); + // if is_selected { + let label = format!("{}{}", std::char::from_u32(letter1 as u32 + 65).unwrap(), std::char::from_u32(letter2 as u32 + 65).unwrap()); + ui.allocate_ui_at_rect(egui::Rect::from_min_max( + egui::pos2(*start_x as f32, *start_y as f32), + egui::pos2(*start_x as f32 + 100.0, *end_y as f32 + 100.0), + ), |ui| { + ui.label(egui::RichText::new(label).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(image_color)); + }); + // } + } + } +} + impl eframe::App for MyApp { fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { let mut key_to_click:Option = None; + let mut close = false; + let mut width = 0; + let mut height = 0; ctx.input(|i| { - if i.key_pressed(egui::Key::Escape) { - std::process::exit(0); + width = i.screen_rect().width() as i32; + height = i.screen_rect().height() as i32; + if i.key_released(egui::Key::Escape) { + close = true; } - if i.key_pressed(egui::Key::Backspace) { + if i.key_released(egui::Key::Backspace) { self.letters_typed.pop(); } - if i.key_pressed(egui::Key::Num1) - || i.key_pressed(egui::Key::Enter) - || i.key_pressed(egui::Key::Num2) - || i.key_pressed(egui::Key::Num3) - || i.key_pressed(egui::Key::Num4) { + if i.key_released(egui::Key::Num1) + || i.key_released(egui::Key::Enter) + || i.key_released(egui::Key::Num2) + || i.key_released(egui::Key::Num3) + || i.key_released(egui::Key::Num4) { - if i.key_pressed(egui::Key::Num1) || i.key_pressed(egui::Key::Enter) { + if i.key_released(egui::Key::Num1) || i.key_released(egui::Key::Enter) { key_to_click = Some(ClickType::Left); - } else if i.key_pressed(egui::Key::Num2) { - key_to_click = Some(ClickType::Right); - } else if i.key_pressed(egui::Key::Num3) { - key_to_click = Some(ClickType::Middle); - } else { - key_to_click = Some(ClickType::Double); } + // else if i.key_pressed(egui::Key::Num2) { + // key_to_click = Some(ClickType::Right); + // } else if i.key_pressed(egui::Key::Num3) { + // key_to_click = Some(ClickType::Middle); + // } else { + // key_to_click = Some(ClickType::Double); + // } } let key = get_key(i); match key { Some(key) => { println!("Key pressed: {:?}", key); - self.letters_typed.append(&mut vec![key as u32]); + // max len 3 + if self.letters_typed.len() < 3 { + self.letters_typed.append(&mut vec![key as u32]); + } } None => {} } @@ -160,17 +314,59 @@ impl eframe::App for MyApp { std::process::exit(0); }); } + if close { + ctx.send_viewport_cmd(egui::ViewportCommand::Close); + } + + // if self.letters_typed.len() == 2 { + // let index = get_index_for_letters(self.letters_typed[0] as u8, self.letters_typed[1] as u8); + // // ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true)); + // let (min_x, min_y, max_x, max_y) = if index < self.line_boxes.len() as i32 { + // self.line_boxes[index as usize] + // } else { + // self.big_boxes[(index - self.line_boxes.len() as i32) as usize] + // }; + // let width = self.width; + // let height = self.height; + // tokio::spawn(async move { + // // click to center of box + // let click_x = (min_x + max_x) / 2; + // let click_y = (min_y + max_y) / 2; + + // autotype::movemouse(click_x as i32, click_y as i32, width as i32, height as i32).await.unwrap(); + // }); + + // } + + ui.add( + egui::Image::new("file://".to_owned() + &self.path) + ); + + self.draw_text_lines(ui); + self.draw_big_boxes(ui); + self.draw_images(ui); + if self.selected_box.is_some() { + let (min_x, min_y, max_x, max_y) = self.selected_box.unwrap(); + + // draw half transparent box + ui.painter().rect( + egui::Rect::from_min_max( + egui::pos2(min_x as f32, min_y as f32), + egui::pos2(max_x as f32, max_y as f32), + ), + 0.0, + egui::Color32::from_rgba_premultiplied(255, 0, 150, 5), + egui::Stroke::default() + ); + ui.painter().rect_stroke( + egui::Rect::from_min_max( + egui::pos2(min_x as f32, min_y as f32), + egui::pos2(max_x as f32, max_y as f32), + ), + 0.0, + egui::Stroke::new(4.0, egui::Color32::from_rgb(255, 255, 255)), + ); - if self.letters_typed.len() == 2 { - let index = get_index_for_letters(self.letters_typed[0] as u8, self.letters_typed[1] as u8); - // ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true)); - let (min_x, min_y, max_x, max_y) = if index < self.positions.len() as i32 { - self.positions[index as usize] - } else { - self.big_boxes[(index - self.positions.len() as i32) as usize] - }; - let width = self.width; - let height = self.height; tokio::spawn(async move { // click to center of box let click_x = (min_x + max_x) / 2; @@ -178,111 +374,208 @@ impl eframe::App for MyApp { autotype::movemouse(click_x as i32, click_y as i32, width as i32, height as i32).await.unwrap(); }); - } - ui.add( - egui::Image::new("file://".to_owned() + &self.path) - ); - for ((min_x, min_y, max_x, max_y), index) in self.big_boxes.iter().zip(0..) { - let box_index = index as u32 + self.positions.len() as u32; - let (letter1, letter2) = get_letters_for_index(box_index as i32); - let letter1char = std::char::from_u32(letter1 as u32 + 65).unwrap(); - let letter2char = std::char::from_u32(letter2 as u32 + 65).unwrap(); - - if self.letters_typed.len() == 0 || self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == letter1 { - // color magenta if letter2 matches - let color = if self.letters_typed.len() == 2 && self.letters_typed[1] as u8 == letter2 { - // draw half transparent box - ui.painter().rect( - egui::Rect::from_min_max( - egui::pos2(*min_x as f32, *min_y as f32), - egui::pos2(*max_x as f32, *max_y as f32), - ), - 0.0, - egui::Color32::from_rgba_premultiplied(200, 0, 100, 10), - egui::Stroke::default() - ); - egui::Color32::from_rgb(200, 0, 100) - } else { - if self.letters_typed.len() == 2 { - egui::Color32::from_rgb(100, 100, 100) - } else { - egui::Color32::from_rgb(250, 80, 50) - } - }; - ui.painter().rect_stroke( - egui::Rect::from_min_max( - egui::pos2(*min_x as f32, *min_y as f32), - egui::pos2(*max_x as f32, *max_y as f32), - ), - 0.0, - egui::Stroke::new(1.0, color), - ); + // for ((min_x, min_y, max_x, max_y), index) in self.large_images.iter().zip(0..) { + // let box_index = index as u32 + self.line_boxes.len() as u32; + // let (letter1, letter2) = get_letters_for_index(box_index as i32); + // let letter1char = std::char::from_u32(letter1 as u32 + 65).unwrap(); + // let letter2char = std::char::from_u32(letter2 as u32 + 65).unwrap(); + + // if self.letters_typed.len() == 0 || self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == letter1 { + // // color magenta if letter2 matches + // let color = if self.letters_typed.len() == 2 && self.letters_typed[1] as u8 == letter2 { + // // draw half transparent box + // ui.painter().rect( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Color32::from_rgba_premultiplied(200, 0, 100, 10), + // egui::Stroke::default() + // ); + // egui::Color32::from_rgb(200, 0, 100) + // } else { + // if self.letters_typed.len() == 2 { + // egui::Color32::from_rgb(100, 100, 100) + // } else { + // egui::Color32::from_rgb(130, 50, 250) + // } + // }; + // ui.painter().rect_stroke( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Stroke::new(1.0, color), + // ); - ui.allocate_ui_at_rect(egui::Rect::from_min_max( - egui::pos2(*min_x as f32, *min_y as f32), - egui::pos2((*min_x + 50) as f32, (*min_y + 50) as f32), - ), |ui| { - ui.label(egui::RichText::new(format!("{}{}", letter1char, letter2char)).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color)); - }); - } - } + // ui.allocate_ui_at_rect(egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2((*min_x + 50) as f32, (*min_y + 50) as f32), + // ), |ui| { + // ui.label(egui::RichText::new(format!("{}{}", letter1char, letter2char)).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color)); + // }); + // } + // } - for ((min_x, min_y, max_x, max_y), index) in self.positions.iter().zip(0..) { - let (letter1, letter2) = get_letters_for_index(index); - let letter1char = std::char::from_u32(letter1 as u32 + 65).unwrap(); - let letter2char = std::char::from_u32(letter2 as u32 + 65).unwrap(); - - if self.letters_typed.len() == 0 || self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == letter1 { - // color magenta if letter2 matches - let color = if self.letters_typed.len() == 2 && self.letters_typed[1] as u8 == letter2 { - // draw half transparent box - ui.painter().rect( - egui::Rect::from_min_max( - egui::pos2(*min_x as f32, *min_y as f32), - egui::pos2(*max_x as f32, *max_y as f32), - ), - 0.0, - egui::Color32::from_rgba_premultiplied(200, 0, 100, 10), - egui::Stroke::default() - ); - egui::Color32::from_rgb(200, 0, 100) - } else { - if self.letters_typed.len() == 2 { - egui::Color32::from_rgb(100, 100, 100) - } else { - egui::Color32::from_rgb(0, 150, 150) - } - }; - ui.painter().rect_stroke( - egui::Rect::from_min_max( - egui::pos2(*min_x as f32, *min_y as f32), - egui::pos2(*max_x as f32, *max_y as f32), - ), - 0.0, - egui::Stroke::new(1.0, color), - ); + // for ((min_x, min_y, max_x, max_y), index) in self.line_boxes.iter().zip(0..) { + // let box_index = index as u32 + self.line_boxes.len() as u32; + // let (letter1, letter2) = get_letters_for_index(box_index as i32); + // let letter1char = std::char::from_u32(letter1 as u32 + 65).unwrap(); + // let letter2char = std::char::from_u32(letter2 as u32 + 65).unwrap(); + + // if self.letters_typed.len() == 0 || self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == letter1 { + // // color magenta if letter2 matches + // let color = if self.letters_typed.len() == 2 && self.letters_typed[1] as u8 == letter2 { + // // draw half transparent box + // ui.painter().rect( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Color32::from_rgba_premultiplied(200, 0, 100, 10), + // egui::Stroke::default() + // ); + // egui::Color32::from_rgb(200, 0, 100) + // } else { + // if self.letters_typed.len() == 2 { + // egui::Color32::from_rgb(100, 100, 100) + // } else { + // egui::Color32::from_rgb(100,250, 10) + // } + // }; + // ui.painter().rect_stroke( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Stroke::new(1.0, color), + // ); - ui.allocate_ui_at_rect(egui::Rect::from_min_max( - egui::pos2(*min_x as f32, *min_y as f32), - egui::pos2((*min_x + 50) as f32, (*min_y + 50) as f32), - ), |ui| { - ui.label(egui::RichText::new(format!("{}{}", letter1char, letter2char)).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color)); - }); - } + // ui.allocate_ui_at_rect(egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2((*min_x + 50) as f32, (*min_y + 50) as f32), + // ), |ui| { + // ui.label(egui::RichText::new(format!("{}{}", letter1char, letter2char)).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color)); + // }); + // } + // } + + // for ((min_x, min_y, max_x, max_y), index) in self.big_boxes.iter().zip(0..) { + // let box_index = index as u32 + self.line_boxes.len() as u32; + // let (letter1, letter2) = get_letters_for_index(box_index as i32); + // let letter1char = std::char::from_u32(letter1 as u32 + 65).unwrap(); + // let letter2char = std::char::from_u32(letter2 as u32 + 65).unwrap(); + + // if self.letters_typed.len() == 0 || self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == letter1 { + // // color magenta if letter2 matches + // let color = if self.letters_typed.len() == 2 && self.letters_typed[1] as u8 == letter2 { + // // draw half transparent box + // ui.painter().rect( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Color32::from_rgba_premultiplied(200, 0, 100, 10), + // egui::Stroke::default() + // ); + // egui::Color32::from_rgb(200, 0, 100) + // } else { + // if self.letters_typed.len() == 2 { + // egui::Color32::from_rgb(100, 100, 100) + // } else { + // egui::Color32::from_rgb(250, 80, 50) + // } + // }; + // ui.painter().rect_stroke( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Stroke::new(1.0, color), + // ); + + // ui.allocate_ui_at_rect(egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2((*min_x + 50) as f32, (*min_y + 50) as f32), + // ), |ui| { + // ui.label(egui::RichText::new(format!("{}{}", letter1char, letter2char)).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color)); + // }); + // } + // } + + // for ((min_x, min_y, max_x, max_y), index) in self.small_images.iter().zip(0..) { + // let (letter1, letter2) = get_letters_for_index(index); + // let letter1char = std::char::from_u32(letter1 as u32 + 65).unwrap(); + // let letter2char = std::char::from_u32(letter2 as u32 + 65).unwrap(); + + // if self.letters_typed.len() == 0 || self.letters_typed.len() > 0 && self.letters_typed[0] as u8 == letter1 { + // // color magenta if letter2 matches + // let color = if self.letters_typed.len() == 2 && self.letters_typed[1] as u8 == letter2 { + // // draw half transparent box + // ui.painter().rect( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Color32::from_rgba_premultiplied(200, 0, 100, 10), + // egui::Stroke::default() + // ); + // egui::Color32::from_rgb(200, 0, 100) + // } else { + // if self.letters_typed.len() == 2 { + // egui::Color32::from_rgb(100, 100, 100) + // } else { + // egui::Color32::from_rgb(0, 150, 150) + // } + // }; + // ui.painter().rect_stroke( + // egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2(*max_x as f32, *max_y as f32), + // ), + // 0.0, + // egui::Stroke::new(1.0, color), + // ); + + // ui.allocate_ui_at_rect(egui::Rect::from_min_max( + // egui::pos2(*min_x as f32, *min_y as f32), + // egui::pos2((*min_x + 50) as f32, (*min_y + 50) as f32), + // ), |ui| { + // ui.label(egui::RichText::new(format!("{}{}", letter1char, letter2char)).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color)); + // }); + // } ui.allocate_ui_at_rect(egui::Rect::from_min_size( egui::pos2(0.0, 0.0), egui::vec2(200.0, 100.0), ), |ui| { + if self.letters_typed.len() == 0 { + return + } // combine vec let letters = self.letters_typed.iter().map(|x| std::char::from_u32(*x as u32 + 65).unwrap()).collect::(); + // color by letter + let color = if self.letters_typed[0] == 8 { + egui::Color32::from_rgb(255, 160, 50) + } else if self.letters_typed[0] == 11 { + egui::Color32::from_rgb(150, 200, 20) + } else { + egui::Color32::from_rgb(50, 100, 200) + }; - ui.label(egui::RichText::new(letters).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(egui::Color32::from_rgb(50, 100, 200)).size(40.0)); + ui.label(egui::RichText::new(letters).heading().color(egui::Color32::from_rgb(255, 255, 255)).background_color(color).size(40.0)); }); - } + // } }); } } diff --git a/src/image_utils.rs b/src/image_utils.rs index 3f15cf6..c507ed1 100644 --- a/src/image_utils.rs +++ b/src/image_utils.rs @@ -1,21 +1,21 @@ use image::{DynamicImage, GenericImage}; -pub fn draw_box(image: &mut DynamicImage, min_x: u32, min_y: u32, max_x: u32, max_y: u32, color: image::Rgba) { - for x in min_x..max_x { - for y in min_y..max_y { - if x >= image.width() || y >= image.height() { +pub fn draw_box(image: &mut DynamicImage, min_x: usize, min_y: usize, max_x: usize, max_y: usize, color: image::Rgba) { + for x in min_x..(max_x+1) { + for y in min_y..(max_y+1) { + if x >= image.width() as usize || y >= image.height() as usize { continue; } - if x == min_x || x == max_x-1 || y == min_y || y == max_y-1 { - image.put_pixel(x, y, color); + if x == min_x || x == max_x || y == min_y || y == max_y { + image.put_pixel(x as u32, y as u32, color); } } } } -pub fn draw_boxes(image: &mut DynamicImage, boxes: &[(u32, u32, u32, u32)]) { +pub fn draw_boxes(image: &mut DynamicImage, boxes: &[(usize, usize, usize, usize)], color: image::Rgba) { for (min_x, min_y, max_x, max_y) in boxes { - draw_box(image, *min_x, *min_y, *max_x, *max_y, image::Rgba([255, 0, 0, 255])); + draw_box(image, *min_x, *min_y, *max_x, *max_y, color); } } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..86a01f4 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,6 @@ +pub mod autotype; +pub mod gui; +pub mod image_utils; +pub mod bounding_box; +pub mod globalshortcut; +pub mod screenshot; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index c8337bc..0000000 --- a/src/main.rs +++ /dev/null @@ -1,43 +0,0 @@ -#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] -#![allow(rustdoc::missing_crate_level_docs)] - -mod bounding_box; -mod gui; -mod screenshot; -mod autotype; -mod image_utils; - -const SCREENSHOT_PATH: &str = "/tmp/screenshot.png"; - -#[tokio::main] -async fn main() { - println!("[Main] Taking screenshot"); - let start_time = std::time::Instant::now(); - let screenshot_uri = match screenshot::screenshot().await { - Ok(uri) => uri, - Err(err) => { - println!("[Main] Failed to take screenshot: {:?}", err); - return; - } - }; - // let screenshot_uri = "/home/quexten/screenshot_5.png"; - println!("[Main] Elapsed: {:?}", start_time.elapsed()); - - println!("[Main] Opening screenshot"); - let start_time = std::time::Instant::now(); - let screenshot = image::open(screenshot_uri.clone()).unwrap(); - std::fs::copy(screenshot_uri.clone(), SCREENSHOT_PATH).unwrap(); - // std::fs::remove_file(screenshot_uri).unwrap(); - println!("[Main] Elapsed: {:?}", start_time.elapsed()); - - println!("[Main] Finding bounding boxes"); - let start_time = std::time::Instant::now(); - let (text_boxes, big_boxes)/* bounding_boxes */ = bounding_box::find_bounding_boxes_v2(&screenshot); - println!("[Main] Elapsed: {:?}", start_time.elapsed()); - println!("[Main] Found {:?} small boxes and {:?} big boxes", text_boxes.len(), big_boxes.len()); - - - println!("[Main] Showing GUI"); - autotype::start_autoclick_session().await.unwrap(); - gui::show_gui(text_boxes, big_boxes, screenshot.width(), screenshot.height(), SCREENSHOT_PATH.to_string()); -} diff --git a/src/screenshot.rs b/src/screenshot.rs deleted file mode 100644 index d2c4399..0000000 --- a/src/screenshot.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::fmt::Error; - -use ashpd::desktop::screenshot; - -pub async fn screenshot() -> Result> { - match screenshot::ScreenshotRequest::default() - .interactive(false) - .modal(true) - .send() - .await - .and_then(|r| r.response()) - { - Ok(response) => { - let screenshot = response.uri().clone(); - println!("[Screenshot] Screenshot taken: {}", screenshot); - return Ok(String::from(screenshot.to_string().replace("file://", ""))); - } - Err(err) => { - println!("[Screenshot] Failed to take screenshot: {}", err); - return Err(Box::new(Error)); - } - } -} \ No newline at end of file diff --git a/src/screenshot/mod.rs b/src/screenshot/mod.rs new file mode 100644 index 0000000..851b0f5 --- /dev/null +++ b/src/screenshot/mod.rs @@ -0,0 +1,330 @@ +use std::{borrow::{Borrow, BorrowMut}, fmt::Error, sync::Arc, thread, time::Duration}; + +use ashpd::desktop::{print, screenshot}; +use scap::{ + capturer::{Area, Capturer, Options, Point, Size}, + frame::Frame, +}; +use tokio::{net::unix::pipe::{self, pipe}, sync::Mutex, time::timeout}; +use xcap::Monitor; + +pub struct PipewireCapturer { + needs_screenshot: Arc>, + image_rx: tokio::sync::mpsc::Receiver, Vec>>, +} + +impl PipewireCapturer { + pub async fn take_screenshot(&mut self) -> Result, Vec>, Box> { + let mut needs_capture = self.needs_screenshot.lock().await; + *needs_capture = true; + println!("Needs capture: {:?}", *needs_capture); + drop(needs_capture); + println!("waiting for screenshot"); + let screenshot = timeout(Duration::from_secs(5), self.image_rx.recv()).await; + match screenshot { + Ok(res) => { + match res { + Some(screenshot) => { + return Ok(screenshot); + } + None => { + println!("[PipewireCapturer] Failed to take screenshot"); + return Err(Box::new(Error)); + } + } + } + Err(_) => { + println!("[PipewireCapturer] Timeout"); + return Err(Box::new(Error)); + } + } + } +} + +pub struct ScreenshotTool { + pipewire_capturer: Arc>>, + heartbeat_rx: Arc>>>, +} + +impl ScreenshotTool { + + pub fn start(&mut self) { + let timeout_rx = self.heartbeat_rx.clone(); + let pipewire_capture = self.pipewire_capturer.clone(); + tokio::spawn(async move { + loop { + let mut timeout_rx_opt = timeout_rx.lock().await; + let timeout_rx_opt1 = timeout_rx_opt.borrow_mut(); + let rx = timeout_rx_opt1.as_mut(); + let mut close = false; + match rx { + Some(rx) => { + if rx.is_closed() { + close = true; + } else { + match timeout(Duration::from_secs(10), rx.recv()).await { + Ok(_) => { + } + Err(_) => { + println!("[Watchdog] timeout"); + let mut pipewire_capturer = pipewire_capture.lock().await; + *pipewire_capturer = None; + let mut timeout_rx = timeout_rx.lock().await; + *timeout_rx = None; + } + } + } + } + None => { + println!("[Watchdog] No capturer found"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + drop(timeout_rx_opt); + + if close { + println!("[Watchdog] Capturer closing"); + let mut pipewire_capturer = pipewire_capture.lock().await; + *pipewire_capturer = None; + let mut timeout_rx = timeout_rx.lock().await; + *timeout_rx = None; + println!("Capturer closed"); + } + } + }); + } + + pub async fn take_screenshot(&mut self) -> Result, Vec>, Box> { + match self.take_screenshot_pipewire().await { + Ok(screenshot) => { + println!("[Screenshot Tool] Screenshot taken using pipewire"); + return Ok(screenshot); + } + Err(err) => { + println!("Failed to take screenshot using pipewire: {:?}", err); + println!("[Screenshot Tool] Failed to take screenshot using pipewire, falling back to screenshot portal"); + match screenshot_portal().await { + Ok(screenshot) => { + let screenshot = image::open(screenshot)?.to_rgb8(); + return Ok(screenshot); + } + Err(_) => { + println!("[Screenshot Tool] Failed to take screenshot using screenshot portal"); + let screenshot = screenshot_xcap().await?; + let screenshot = image::open(screenshot)?.to_rgb8(); + return Ok(screenshot); + } + } + } + } + } + + async fn start_capturer_if_needed(&mut self) -> Result<(), Box> { + let heartbeat_rx = self.heartbeat_rx.lock().await; + let is_none = heartbeat_rx.is_none(); + drop(heartbeat_rx); + + println!("Is none: {:?}", is_none); + + if is_none { + let (capturer, rx) = self.start_screenshare().await?; + let mut pipewire_capturer = self.pipewire_capturer.lock().await; + *pipewire_capturer = Some(capturer); + let mut heartbeat_rx = self.heartbeat_rx.lock().await; + *heartbeat_rx = Some(rx); + } + + return Ok(()); + } + + async fn take_screenshot_pipewire(&mut self) -> Result, Vec>, Box> { + println!("Taking screenshot using pipewire"); + self.start_capturer_if_needed().await?; + println!("Capturer started"); + + match self.pipewire_capturer.lock().await.as_mut() { + Some(capturer) => { + let screenshot = capturer.take_screenshot().await?; + return Ok(screenshot); + } + None => { + return Err(Box::new(Error)); + } + } + } + + pub async fn start_screenshare(&mut self) -> Result<(PipewireCapturer, tokio::sync::mpsc::Receiver<()>), Box> { + if !scap::is_supported() { + println!("❌ Platform not supported"); + return Err(Box::new(Error)); + } + if !scap::has_permission() { + println!("❌ Permission not granted. Requesting permission..."); + if !scap::request_permission() { + println!("❌ Permission denied"); + return Err(Box::new(Error)); + } + } + let options = Options { + fps: 60, + excluded_targets: None, + excluded_windows: None, + show_cursor: true, + show_highlight: false, + output_type: scap::frame::FrameType::RGB, + output_resolution: scap::capturer::Resolution::_720p, + source_rect: Some(Area { + origin: Point { x: 0.0, y: 0.0 }, + size: Size { + width: 2000.0, + height: 1000.0, + }, + }), + ..Default::default() + }; + + let needs_capture = Arc::new(Mutex::new(false)); + let (tx, rx) = tokio::sync::mpsc::channel::, Vec>>(1); + let (heartbeat_tx, heartbeat_rx) = tokio::sync::mpsc::channel::<()>(1); + + // self.needs_screenshot = Some(needs_capture.clone()); + let pipewire_capturer = PipewireCapturer { + needs_screenshot: needs_capture.clone(), + image_rx: rx, + }; + + tokio::spawn(async move { + let mut capturer: Capturer = Capturer::new(options); + println!("[Screencapture Thread] Starting capture"); + capturer.start_capture(); + println!("Capture started"); + let mut last_screenshot_taken = std::time::Instant::now(); + let timeout_duration = Duration::from_secs(120); + loop { + if last_screenshot_taken.elapsed() > timeout_duration { + println!("Stopping capturer"); + // close tx + capturer.stop_capture(); + return; + } + + if let Err(_) = timeout(Duration::from_secs(2), heartbeat_tx.send(())).await.unwrap() { + println!("Failed to send heartbeat"); + capturer.stop_capture(); + return; + } + let start = std::time::Instant::now(); + let frame = capturer.get_next_frame(); + let frame = match frame { + Ok(frame) => frame, + Err(_) => { + println!("Failed to get frame"); + return + } + }; + + if start.elapsed().as_millis() > 1000 { + println!("Frame took too long to receive"); + } + + let should_read = needs_capture.lock().await.clone(); + if !should_read { + continue; + } + last_screenshot_taken = std::time::Instant::now(); + println!("[Screencapture Thread] Reading frame"); + let mut should_read = needs_capture.as_ref().lock().await; + println!("Should read: {:?}", *should_read); + *should_read = false; + println!("Should read: {:?}", *should_read); + println!("[Screencapture Thread] Frame read"); + + match frame { + Frame::BGRA(frame) => { + println!("BGRA Frame"); + } + Frame::BGR0(frame) => { + println!("BGR0 Frame"); + } + Frame::RGB(frame) => { + println!("RGB Frame"); + } + Frame::RGBx(frame) => { + println!("RGBx Frame"); + } + Frame::XBGR(frame) => { + println!("XBGR Frame"); + } + Frame::BGRx(frame) => { + // empty frame black with same size + let mut image = image::ImageBuffer::from_fn(frame.width as u32, frame.height as u32, |x, y| { + image::Rgb([0, 0, 0]) + }); + let start = std::time::Instant::now(); + frame.data.chunks_exact(4).enumerate().for_each(|(i, pixel)| { + let x = i % frame.width as usize; + let y = i / frame.width as usize; + let pixel = image::Rgb([pixel[2], pixel[1], pixel[0]]); + image.put_pixel(x as u32, y as u32, pixel); + }); + println!("Image creation: {:?}", start.elapsed()); + println!("Sending image"); + tx.send(image.clone()).await.unwrap(); + } + Frame::YUVFrame(frame) => { + println!("YUV Frame"); + } + _ => { + println!("Frame type not supported"); + } + } + } + }); + + return Ok((pipewire_capturer, heartbeat_rx)); + } +} + +pub fn get_screenshot_tool() -> ScreenshotTool { + let mut screenshot_tool = ScreenshotTool { + pipewire_capturer: Arc::new(Mutex::new(None)), + heartbeat_rx: Arc::new(Mutex::new(None)), + }; + screenshot_tool.start(); + return screenshot_tool; +} + +pub async fn screenshot_portal() -> Result> { + match screenshot::ScreenshotRequest::default() + .interactive(false) + .modal(true) + .send() + .await + .and_then(|r| r.response()) + { + Ok(response) => { + let screenshot = response.uri().clone(); + println!("[Scoeeenshot] Screenshot taken: {}", screenshot); + return Ok(String::from(screenshot.to_string().replace("file://", ""))); + } + Err(err) => { + println!("[Screenshot] Failed to take screenshot: {}", err); + return Err(Box::new(Error)); + } + } +} + +pub async fn screenshot_xcap() -> Result> { + let monitors = Monitor::all().unwrap(); + let monitor = monitors.get(0); + match monitor { + Some(monitor) => { + let res = monitor.capture_image()?; + res.save("/tmp/tmp.png")?; + return Ok(String::from("/tmp/tmp.png")); + } + None => { + return Err(Box::new(Error)); + } + } +} \ No newline at end of file diff --git a/swiftmouse.svg b/swiftmouse.svg new file mode 100644 index 0000000..0c299be --- /dev/null +++ b/swiftmouse.svg @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + +