diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f084a5e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/psf/black + rev: 22.8.0 + hooks: + - id: black + language_version: python3.11 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bbf838..63f980a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +## [0.1.0] - 2023-01-12 + +### Added + +- Added full-text search for keys, with pagination using `store.search(term, skip, limit)` + +### Changed + +- Changed the `Store::new()` signature to include `max_search_index_key_length` option. + +### Fixed + ## [0.0.3] - 2022-11-09 ### Added diff --git a/Cargo.lock b/Cargo.lock index f59cf1d..0699224 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ - "libc", + "libc", ] [[package]] @@ -17,9 +17,9 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", + "concurrent-queue", + "event-listener", + "futures-core", ] [[package]] @@ -28,12 +28,12 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "once_cell", - "slab", + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", ] [[package]] @@ -42,13 +42,13 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca" dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", ] [[package]] @@ -57,18 +57,18 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7" dependencies = [ - "autocfg", - "concurrent-queue", - "futures-lite", - "libc", - "log", - "once_cell", - "parking", - "polling", - "slab", - "socket2", - "waker-fn", - "winapi", + "autocfg", + "concurrent-queue", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "winapi", ] [[package]] @@ -77,8 +77,8 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" dependencies = [ - "event-listener", - "futures-lite", + "event-listener", + "futures-lite", ] [[package]] @@ -87,16 +87,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02111fd8655a613c25069ea89fc8d9bb89331fa77486eb3bc059ee757cfa481c" dependencies = [ - "async-io", - "autocfg", - "blocking", - "cfg-if", - "event-listener", - "futures-lite", - "libc", - "once_cell", - "signal-hook", - "winapi", + "async-io", + "autocfg", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "libc", + "once_cell", + "signal-hook", + "winapi", ] [[package]] @@ -105,25 +105,25 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "async-process", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", ] [[package]] @@ -156,12 +156,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" dependencies = [ - "async-channel", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", - "once_cell", + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", ] [[package]] @@ -194,22 +194,19 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "iana-time-zone", - "js-sys", - "num-integer", - "num-traits", - "time", - "wasm-bindgen", - "winapi", + "iana-time-zone", + "num-integer", + "num-traits", + "winapi", ] [[package]] name = "clokwerk" -version = "0.3.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a99373edf782eec8fde0d52bd2f60e4ae5eaca31f1d2cf5aec68243849a37a9" +checksum = "bd108d365fcb6d7eddf17a6718eb6a33db18ba4178f8cc6b667f480710f10d76" dependencies = [ - "chrono", + "chrono", ] [[package]] @@ -218,8 +215,8 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ - "termcolor", - "unicode-width", + "termcolor", + "unicode-width", ] [[package]] @@ -228,7 +225,7 @@ version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" dependencies = [ - "cache-padded", + "cache-padded", ] [[package]] @@ -243,7 +240,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" dependencies = [ - "cfg-if", + "cfg-if", ] [[package]] @@ -252,8 +249,8 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote", - "syn", + "quote", + "syn", ] [[package]] @@ -262,10 +259,10 @@ version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", ] [[package]] @@ -274,13 +271,13 @@ version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", ] [[package]] @@ -295,9 +292,9 @@ version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -312,7 +309,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" dependencies = [ - "instant", + "instant", ] [[package]] @@ -321,13 +318,13 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", ] [[package]] @@ -336,8 +333,8 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ - "futures-core", - "futures-sink", + "futures-core", + "futures-sink", ] [[package]] @@ -352,9 +349,9 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ - "futures-core", - "futures-task", - "futures-util", + "futures-core", + "futures-task", + "futures-util", ] [[package]] @@ -369,13 +366,13 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", ] [[package]] @@ -384,9 +381,9 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -407,16 +404,16 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", ] [[package]] @@ -425,9 +422,9 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "cfg-if", + "libc", + "wasi", ] [[package]] @@ -436,10 +433,10 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9" dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -448,12 +445,12 @@ version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "winapi", + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", ] [[package]] @@ -462,8 +459,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" dependencies = [ - "cxx", - "cxx-build", + "cxx", + "cxx-build", ] [[package]] @@ -478,7 +475,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if", ] [[package]] @@ -487,7 +484,7 @@ version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ - "wasm-bindgen", + "wasm-bindgen", ] [[package]] @@ -496,7 +493,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ - "log", + "log", ] [[package]] @@ -511,7 +508,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" dependencies = [ - "cc", + "cc", ] [[package]] @@ -520,8 +517,8 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ - "autocfg", - "scopeguard", + "autocfg", + "scopeguard", ] [[package]] @@ -530,8 +527,8 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if", - "value-bag", + "cfg-if", + "value-bag", ] [[package]] @@ -546,7 +543,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ - "autocfg", + "autocfg", ] [[package]] @@ -555,8 +552,8 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg", - "num-traits", + "autocfg", + "num-traits", ] [[package]] @@ -565,7 +562,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg", + "autocfg", ] [[package]] @@ -586,8 +583,8 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api", + "parking_lot_core", ] [[package]] @@ -596,11 +593,11 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys", + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", ] [[package]] @@ -621,12 +618,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" dependencies = [ - "autocfg", - "cfg-if", - "libc", - "log", - "wepoll-ffi", - "winapi", + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", ] [[package]] @@ -641,17 +638,17 @@ version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ - "unicode-ident", + "unicode-ident", ] [[package]] name = "py_scdb" version = "0.1.0" dependencies = [ - "async-std", - "pyo3", - "pyo3-asyncio", - "scdb", + "async-std", + "pyo3", + "pyo3-asyncio", + "scdb", ] [[package]] @@ -660,15 +657,15 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201b6887e5576bf2f945fe65172c1fcbf3fcf285b23e4d71eb171d9736e38d32" dependencies = [ - "cfg-if", - "indoc", - "libc", - "memoffset", - "parking_lot", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", ] [[package]] @@ -677,12 +674,12 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1febe3946b26194628f00526929ee6f8559f9e807f811257e94d4c456103be0e" dependencies = [ - "async-std", - "futures", - "once_cell", - "pin-project-lite", - "pyo3", - "pyo3-asyncio-macros", + "async-std", + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "pyo3-asyncio-macros", ] [[package]] @@ -691,9 +688,9 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "270b167ea304b961616aa0f003463c95becd3c11a85bd83ba668fe16129e2469" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -702,8 +699,8 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf0708c9ed01692635cbf056e286008e5a2927ab1a5e48cdd3aeb1ba5a6fef47" dependencies = [ - "once_cell", - "target-lexicon", + "once_cell", + "target-lexicon", ] [[package]] @@ -745,7 +742,7 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ - "proc-macro2", + "proc-macro2", ] [[package]] @@ -754,9 +751,9 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "libc", - "rand_chacha", - "rand_core", + "libc", + "rand_chacha", + "rand_core", ] [[package]] @@ -765,8 +762,8 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ - "ppv-lite86", - "rand_core", + "ppv-lite86", + "rand_core", ] [[package]] @@ -775,7 +772,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom", ] [[package]] @@ -784,19 +781,20 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags", ] [[package]] name = "scdb" -version = "0.0.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650bf2ff331084b5a8182c35ae0d146888141a9a55692f72bb161dc479f89601" +checksum = "fe2879d89c7450edf01522a9846be87fa3f396044b3da32bd38de0ae8f51ad8c" dependencies = [ - "clokwerk", - "libc", - "twox-hash", - "winapi", + "clokwerk", + "libc", + "memchr", + "twox-hash", + "winapi", ] [[package]] @@ -817,8 +815,8 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" dependencies = [ - "libc", - "signal-hook-registry", + "libc", + "signal-hook-registry", ] [[package]] @@ -827,7 +825,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ - "libc", + "libc", ] [[package]] @@ -836,7 +834,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" dependencies = [ - "autocfg", + "autocfg", ] [[package]] @@ -851,8 +849,8 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ - "libc", - "winapi", + "libc", + "winapi", ] [[package]] @@ -867,9 +865,9 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] @@ -884,18 +882,7 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ - "winapi-util", -] - -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "winapi-util", ] [[package]] @@ -904,9 +891,9 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if", - "rand", - "static_assertions", + "cfg-if", + "rand", + "static_assertions", ] [[package]] @@ -933,8 +920,8 @@ version = "1.0.0-alpha.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ - "ctor", - "version_check", + "ctor", + "version_check", ] [[package]] @@ -949,12 +936,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -967,8 +948,8 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", + "cfg-if", + "wasm-bindgen-macro", ] [[package]] @@ -977,13 +958,13 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", ] [[package]] @@ -992,10 +973,10 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1004,8 +985,8 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ - "quote", - "wasm-bindgen-macro-support", + "quote", + "wasm-bindgen-macro-support", ] [[package]] @@ -1014,11 +995,11 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", ] [[package]] @@ -1033,8 +1014,8 @@ version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ - "js-sys", - "wasm-bindgen", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -1043,7 +1024,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" dependencies = [ - "cc", + "cc", ] [[package]] @@ -1052,8 +1033,8 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] @@ -1068,7 +1049,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi", + "winapi", ] [[package]] @@ -1083,13 +1064,13 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2f24786..230c41f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,6 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.17", features = ["extension-module"] } -scdb = "0.0" +scdb = "0.1" pyo3-asyncio = { version = "0.17", features = ["attributes", "async-std-runtime"] } async-std = "1.12" diff --git a/README.md b/README.md index c397131..9d4fe1a 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,9 @@ if __name__ == "__main__": max_keys=1000000, redundant_blocks=1, pool_capacity=10, - compaction_interval=1800) + compaction_interval=1800, + max_index_key_len=3, + ) # inserting without ttl for (k, v) in records[:3]: @@ -86,6 +88,14 @@ if __name__ == "__main__": for k in keys: v = store.get(k=k) print(f"Key: {k}, Value: {v}") + + # searching without pagination + results = store.search(term="h") + print(f"Search 'h' (no pagination):\n{results}\n") + + # searching with pagination + results = store.search(term="h", skip=1, limit=2) + print(f"Search 'h' (skip=1, limit=2):\n{results}\n") # deleting for k in keys[:3]: @@ -137,7 +147,9 @@ async def run_async_example(): max_keys=1000000, redundant_blocks=1, pool_capacity=10, - compaction_interval=1800) + compaction_interval=1800, + max_index_key_len=3, + ) # inserting without ttl for (k, v) in records[:3]: @@ -162,6 +174,14 @@ async def run_async_example(): for k in keys: v = await store.get(k=k) print(f"Key: {k}, Value: {v}") + + # searching without pagination + results = await store.search(term="h") + print(f"Search 'h' (no pagination):\n{results}\n") + + # searching with pagination + results = await store.search(term="h", skip=1, limit=2) + print(f"Search 'h' (skip=1, limit=2):\n{results}\n") # deleting for k in keys[:3]: @@ -196,6 +216,8 @@ folder of the [rust scdb](https://github.com/sopherapps/scdb) to get up to speed - [database file format](https://github.com/sopherapps/scdb/tree/master/docs/DB_FILE_FORMAT.md) - [how it works](https://github.com/sopherapps/scdb/tree/master/docs/HOW_IT_WORKS.md) +- [inverted index file format](https://github.com/sopherapps/scdb/tree/master/docs/INVERTED_INDEX_FILE_FORMAT.md) +- [how the search works](https://github.com/sopherapps/scdb/tree/master/docs/HOW_INVERTED_INDEX_WORKS.md) ## Bindings @@ -256,38 +278,62 @@ pytest test/test_benchmarks.py --benchmark-columns=mean,min,max --benchmark-name ## Benchmarks -On an average PC. +On an average PC (17Core, 16 GB RAM) ### Synchronous ``` ----------------------------------------------------- benchmark: 23 tests ----------------------------------------------------- -Name (time in ns) Mean Min Max ------------------------------------------------------------------------------------------------------------------------------- -benchmark_get[sync_store-oi] 791.6189 (1.02) 696.0000 (1.0) 82,662.0000 (3.66) -benchmark_get[sync_store-salut] 785.4837 (1.01) 705.0000 (1.01) 26,959.0000 (1.19) -benchmark_get[sync_store-hey] 784.9184 (1.01) 706.0000 (1.01) 49,103.0000 (2.17) -benchmark_get[sync_store-bonjour] 788.7490 (1.01) 707.0000 (1.02) 38,032.0000 (1.68) -benchmark_get[sync_store-mulimuta] 813.3644 (1.04) 710.0000 (1.02) 43,976.0000 (1.95) -benchmark_get[sync_store-hola] 778.5139 (1.0) 717.0000 (1.03) 22,595.0000 (1.0) -benchmark_get[sync_store-hi] 799.0443 (1.03) 720.0000 (1.03) 41,808.0000 (1.85) -benchmark_delete[sync_store-oi] 3,621.6866 (4.65) 3,411.0000 (4.90) 79,191.0000 (3.50) -benchmark_delete[sync_store-mulimuta] 3,692.2154 (4.74) 3,452.0000 (4.96) 65,033.0000 (2.88) -benchmark_delete[sync_store-hi] 3,672.9944 (4.72) 3,464.0000 (4.98) 53,222.0000 (2.36) -benchmark_delete[sync_store-hola] 3,662.2025 (4.70) 3,484.0000 (5.01) 53,862.0000 (2.38) -benchmark_delete[sync_store-hey] 3,692.2129 (4.74) 3,486.0000 (5.01) 59,462.0000 (2.63) -benchmark_delete[sync_store-bonjour] 3,733.3740 (4.80) 3,489.0000 (5.01) 56,492.0000 (2.50) -benchmark_delete[sync_store-salut] 3,703.2190 (4.76) 3,496.0000 (5.02) 67,755.0000 (3.00) -benchmark_set[sync_store-salut-French] 10,501.6776 (13.49) 8,502.0000 (12.22) 3,728,135.0000 (165.00) -benchmark_set[sync_store-hola-Spanish] 10,070.0885 (12.94) 8,619.0000 (12.38) 103,351.0000 (4.57) -benchmark_set[sync_store-bonjour-French] 11,022.1169 (14.16) 8,646.0000 (12.42) 8,880,446.0000 (393.03) -benchmark_set[sync_store-hi-English] 11,689.5849 (15.02) 8,658.0000 (12.44) 21,890,731.0000 (968.83) -benchmark_set[sync_store-oi-Portuguese] 9,977.9048 (12.82) 8,733.0000 (12.55) 71,176.0000 (3.15) -benchmark_set[sync_store-hey-English] 10,487.5903 (13.47) 8,831.0000 (12.69) 2,436,681.0000 (107.84) -benchmark_set[sync_store-mulimuta-Runyoro] 9,909.8211 (12.73) 9,273.0000 (13.32) 101,878.0000 (4.51) -benchmark_clear[sync_store] 99,848.0430 (128.25) 81,451.0000 (117.03) 320,574.0000 (14.19) -benchmark_compact[sync_store] 27,771,153.0357 (>1000.0) 22,487,888.0000 (>1000.0) 36,291,919.0000 (>1000.0) ------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------ benchmark: 47 tests ------------------------------------------------------- +Name (time in ns) Mean Min Max +---------------------------------------------------------------------------------------------------------------------------------- +benchmark_get[sync_store-hey] 800.4672 (1.03) 698.0000 (1.0) 59,185.0000 (1.48) +benchmark_get[sync_store-oi] 776.5698 (1.0) 699.0000 (1.00) 39,861.0000 (1.0) +benchmark_get[sync_store-salut] 785.0628 (1.01) 702.0000 (1.01) 49,960.0000 (1.25) +benchmark_get[sync_store-bonjour] 780.4937 (1.01) 703.0000 (1.01) 71,929.0000 (1.80) +benchmark_get[sync_store-mulimuta] 783.5682 (1.01) 705.0000 (1.01) 50,234.0000 (1.26) +benchmark_get[sync_store-hi] 790.7354 (1.02) 709.0000 (1.02) 70,327.0000 (1.76) +benchmark_get[sync_store-hola] 802.1952 (1.03) 721.0000 (1.03) 61,532.0000 (1.54) +benchmark_paginated_search[sync_store-for] 6,962.4849 (8.97) 6,547.0000 (9.38) 56,507.0000 (1.42) +benchmark_paginated_search[sync_store-pigg] 6,919.7214 (8.91) 6,556.0000 (9.39) 130,593.0000 (3.28) +benchmark_paginated_search[sync_store-bar] 7,079.0014 (9.12) 6,556.0000 (9.39) 86,028.0000 (2.16) +benchmark_paginated_search[sync_store-pi] 6,936.8665 (8.93) 6,556.0000 (9.39) 167,335.0000 (4.20) +benchmark_search[sync_store-pigg] 6,902.7711 (8.89) 6,613.0000 (9.47) 61,975.0000 (1.55) +benchmark_paginated_search[sync_store-pig] 7,028.6412 (9.05) 6,643.0000 (9.52) 89,304.0000 (2.24) +benchmark_paginated_search[sync_store-ban] 7,234.7422 (9.32) 6,653.0000 (9.53) 82,575.0000 (2.07) +benchmark_paginated_search[sync_store-p] 7,027.5200 (9.05) 6,688.0000 (9.58) 58,940.0000 (1.48) +benchmark_search[sync_store-for] 10,294.3237 (13.26) 9,821.0000 (14.07) 71,856.0000 (1.80) +benchmark_search[sync_store-p] 10,391.7488 (13.38) 9,822.0000 (14.07) 111,787.0000 (2.80) +benchmark_search[sync_store-ban] 10,403.6132 (13.40) 9,831.0000 (14.08) 89,734.0000 (2.25) +benchmark_search[sync_store-pig] 10,374.8678 (13.36) 9,841.0000 (14.10) 60,577.0000 (1.52) +benchmark_search[sync_store-pi] 10,380.7388 (13.37) 9,941.0000 (14.24) 108,986.0000 (2.73) +benchmark_search[sync_store-bar] 10,449.2485 (13.46) 9,983.0000 (14.30) 71,575.0000 (1.80) +benchmark_paginated_search[sync_store-fo] 12,587.1956 (16.21) 12,075.0000 (17.30) 69,118.0000 (1.73) +benchmark_paginated_search[sync_store-ba] 13,841.9919 (17.82) 12,089.0000 (17.32) 22,140,639.0000 (555.45) +benchmark_paginated_search[sync_store-f] 12,743.9164 (16.41) 12,150.0000 (17.41) 167,326.0000 (4.20) +benchmark_paginated_search[sync_store-b] 12,762.3290 (16.43) 12,185.0000 (17.46) 141,532.0000 (3.55) +benchmark_paginated_search[sync_store-foo] 12,808.0472 (16.49) 12,268.0000 (17.58) 160,061.0000 (4.02) +benchmark_delete[sync_store-oi] 21,704.5624 (27.95) 13,880.0000 (19.89) 112,458.0000 (2.82) +benchmark_delete[sync_store-hi] 20,088.1860 (25.87) 14,064.0000 (20.15) 93,398.0000 (2.34) +benchmark_delete[sync_store-mulimuta] 21,373.7075 (27.52) 14,105.0000 (20.21) 64,670.0000 (1.62) +benchmark_delete[sync_store-salut] 21,230.0633 (27.34) 14,236.0000 (20.40) 111,376.0000 (2.79) +benchmark_delete[sync_store-hey] 19,834.3945 (25.54) 14,298.0000 (20.48) 81,824.0000 (2.05) +benchmark_delete[sync_store-hola] 21,709.5685 (27.96) 14,315.0000 (20.51) 9,488,572.0000 (238.04) +benchmark_delete[sync_store-bonjour] 21,215.8223 (27.32) 14,406.0000 (20.64) 76,129.0000 (1.91) +benchmark_search[sync_store-ba] 15,359.4292 (19.78) 14,754.0000 (21.14) 127,285.0000 (3.19) +benchmark_search[sync_store-b] 15,462.2251 (19.91) 14,840.0000 (21.26) 74,957.0000 (1.88) +benchmark_search[sync_store-foo] 15,673.4764 (20.18) 14,889.0000 (21.33) 77,673.0000 (1.95) +benchmark_search[sync_store-f] 20,645.8552 (26.59) 19,583.0000 (28.06) 83,335.0000 (2.09) +benchmark_search[sync_store-fo] 20,593.4482 (26.52) 19,716.0000 (28.25) 114,361.0000 (2.87) +benchmark_set[sync_store-hi-English] 26,811.3686 (34.53) 24,591.0000 (35.23) 102,322.0000 (2.57) +benchmark_set[sync_store-oi-Portuguese] 26,792.9735 (34.50) 24,691.0000 (35.37) 105,970.0000 (2.66) +benchmark_set[sync_store-salut-French] 34,000.6656 (43.78) 31,847.0000 (45.63) 114,291.0000 (2.87) +benchmark_set[sync_store-hola-Spanish] 34,219.5426 (44.06) 31,857.0000 (45.64) 103,374.0000 (2.59) +benchmark_set[sync_store-bonjour-French] 34,491.8102 (44.42) 32,046.0000 (45.91) 123,111.0000 (3.09) +benchmark_set[sync_store-mulimuta-Runyoro] 33,995.6793 (43.78) 32,332.0000 (46.32) 117,633.0000 (2.95) +benchmark_set[sync_store-hey-English] 34,103.0846 (43.92) 32,513.0000 (46.58) 107,059.0000 (2.69) +benchmark_clear[sync_store] 168,770.7111 (217.33) 128,008.0000 (183.39) 471,408.0000 (11.83) +benchmark_compact[sync_store] 106,328,709.5000 (>1000.0) 103,646,207.0000 (>1000.0) 108,741,183.0000 (>1000.0) +---------------------------------------------------------------------------------------------------------------------------------- ``` ## Acknowledgement diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 99e46b9..fd0a247 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -55,7 +55,8 @@ People *love* thorough bug reports. I'm not even kidding. ## Use a Consistent Coding Style -* Use [rustfmt](https://github.com/rust-lang/rustfmt) +* Use [rustfmt](https://github.com/rust-lang/rustfmt) for the rust parts +* Use [black](https://github.com/psf/black) for the python parts ## License diff --git a/py_scdb/py_scdb.pyi b/py_scdb/py_scdb.pyi index c994644..3df244b 100644 --- a/py_scdb/py_scdb.pyi +++ b/py_scdb/py_scdb.pyi @@ -1,5 +1,4 @@ -from typing import Optional - +from typing import Optional, List, Tuple class Store: """ @@ -37,14 +36,14 @@ class Store: Default: 3600s (1 hour) """ - def __init__(self, - store_path: str, - max_keys: Optional[int] = None, - redundant_blocks: Optional[int] = None, - pool_capacity: Optional[int] = None, - compaction_interval: Optional[int] = None, - ) -> None: ... - + def __init__( + self, + store_path: str, + max_keys: Optional[int] = None, + redundant_blocks: Optional[int] = None, + pool_capacity: Optional[int] = None, + compaction_interval: Optional[int] = None, + ) -> None: ... def set(self, k: str, v: str, ttl: Optional[int] = None) -> None: """ Inserts or updates the key-value pair @@ -53,7 +52,6 @@ class Store: :param v: the value as a UTF-8 string :param ttl: the number of seconds the key-value pair should be persisted for """ - def get(self, k: str) -> Optional[str]: """ Gets the value associated with the given key @@ -61,19 +59,25 @@ class Store: :param k: the key as a UTF-8 string :return: the value if it exists or None if it doesn't """ + def search(self, term: str, skip: int, limit: int) -> List[Tuple[str, str]]: + """ + Finds all key-values whose keys start with the substring `term`. + :param term: the starting substring to check all keys against + :param skip: the number of the first matched key-value pairs to skip + :param limit: the maximum number of records to return at any one given time + :return: the list of key-value pairs whose key starts with the `term` + """ def delete(self, k: str) -> None: """ Removes the key-value for the given key from the store :param k: the key as a UTF-8 string """ - def clear(self) -> None: """ Removes all data in the store """ - def compact(self) -> None: """ Manually removes dangling key-value pairs in the database file. @@ -94,7 +98,6 @@ class Store: This is a very expensive operation so use it sparingly. """ - class AsyncStore: """ The key-value store that saves key-value pairs (as a UTF-8 string) on disk. @@ -132,14 +135,14 @@ class AsyncStore: Default: 3600s (1 hour) """ - def __init__(self, - store_path: str, - max_keys: Optional[int] = None, - redundant_blocks: Optional[int] = None, - pool_capacity: Optional[int] = None, - compaction_interval: Optional[int] = None, - ) -> None: ... - + def __init__( + self, + store_path: str, + max_keys: Optional[int] = None, + redundant_blocks: Optional[int] = None, + pool_capacity: Optional[int] = None, + compaction_interval: Optional[int] = None, + ) -> None: ... async def set(self, k: str, v: str, ttl: Optional[int] = None) -> None: """ Inserts or updates the key-value pair @@ -148,7 +151,6 @@ class AsyncStore: :param v: the value as a UTF-8 string :param ttl: the number of seconds the key-value pair should be persisted for """ - async def get(self, k: str) -> Optional[str]: """ Gets the value associated with the given key @@ -156,19 +158,25 @@ class AsyncStore: :param k: the key as a UTF-8 string :return: the value if it exists or None if it doesn't """ + async def search(self, term: str, skip: int, limit: int) -> List[Tuple[str, str]]: + """ + Finds all key-values whose keys start with the substring `term`. + :param term: the starting substring to check all keys against + :param skip: the number of the first matched key-value pairs to skip + :param limit: the maximum number of records to return at any one given time + :return: the list of key-value pairs whose key starts with the `term` + """ async def delete(self, k: str) -> None: """ Removes the key-value for the given key from the store :param k: the key as a UTF-8 string """ - async def clear(self) -> None: """ Removes all data in the store """ - async def compact(self) -> None: """ Manually removes dangling key-value pairs in the database file. diff --git a/pyproject.toml b/pyproject.toml index 12b82ab..581b2da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "py_scdb" -version = "0.0.3" +version = "0.1.0" requires-python = ">=3.7" classifiers = [ "Programming Language :: Rust", diff --git a/requirements.txt b/requirements.txt index 32be935..36c9a09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ pytest==7.2.0 pytest-benchmark==4.0.0 pytest-lazy-fixture==0.6.3 pytest-asyncio==0.20.1 +black==22.12.0 +pre-commit==2.21.0 diff --git a/src/async_store.rs b/src/async_store.rs index d3a5116..0a3072a 100644 --- a/src/async_store.rs +++ b/src/async_store.rs @@ -1,5 +1,4 @@ use crate::macros::{acquire_lock, bytes_to_string, io_to_py_result, py_none}; -use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use std::sync::{Arc, Mutex}; @@ -16,7 +15,8 @@ impl AsyncStore { max_keys = "None", redundant_blocks = "None", pool_capacity = "None", - compaction_interval = "None" + compaction_interval = "None", + max_index_key_len = "None" )] #[new] pub fn new( @@ -25,6 +25,7 @@ impl AsyncStore { redundant_blocks: Option, pool_capacity: Option, compaction_interval: Option, + max_index_key_len: Option, ) -> PyResult { let db = io_to_py_result!(scdb::Store::new( store_path, @@ -32,6 +33,7 @@ impl AsyncStore { redundant_blocks, pool_capacity, compaction_interval, + max_index_key_len, ))?; Ok(Self { db: Arc::new(Mutex::new(db)), @@ -85,6 +87,31 @@ impl AsyncStore { ) } + /// Searches for key-values whose key start with the given `term`. + /// + /// In order to do pagination, we use `skip` to skip the first `skip` records + /// and `limit` to return not more than the given number of items + pub fn search<'a>(&mut self, py: Python<'a>, term: &str, skip: u64, limit: u64) -> PyResult<&'a PyAny> { + let locals = pyo3_asyncio::async_std::get_current_locals(py)?; + let db = self.db.clone(); + let term = term.to_owned(); + + pyo3_asyncio::async_std::future_into_py_with_locals( + py, + locals.clone(), + pyo3_asyncio::async_std::scope(locals, async move { + let mut db = acquire_lock!(db)?; + let res = db.search(term.as_bytes(), skip, limit); + let res: Vec<(Vec, Vec)> = io_to_py_result!(res)?; + res.into_iter().map(|(k, v)| { + let k = bytes_to_string!(k)?; + let v = bytes_to_string!(v)?; + Ok((k, v)) + }).collect::>>() + }), + ) + } + /// Deletes the key-value for the given key pub fn delete<'a>(&mut self, py: Python<'a>, k: String) -> PyResult<&'a PyAny> { let locals = pyo3_asyncio::async_std::get_current_locals(py)?; diff --git a/src/store.rs b/src/store.rs index 05548e3..689a78c 100644 --- a/src/store.rs +++ b/src/store.rs @@ -14,7 +14,8 @@ impl Store { max_keys = "None", redundant_blocks = "None", pool_capacity = "None", - compaction_interval = "None" + compaction_interval = "None", + max_index_key_len = "None" )] #[new] pub fn new( @@ -23,6 +24,7 @@ impl Store { redundant_blocks: Option, pool_capacity: Option, compaction_interval: Option, + max_index_key_len: Option, ) -> PyResult { let db = io_to_py_result!(scdb::Store::new( store_path, @@ -30,6 +32,7 @@ impl Store { redundant_blocks, pool_capacity, compaction_interval, + max_index_key_len, ))?; Ok(Self { db }) } @@ -53,6 +56,20 @@ impl Store { } } + /// Searches for key-values whose key start with the given `term`. + /// + /// In order to do pagination, we use `skip` to skip the first `skip` records + /// and `limit` to return not more than the given number of items + pub fn search(&mut self, term: &str, skip: u64, limit: u64) -> PyResult> { + let res = self.db.search(term.as_bytes(), skip, limit); + let res: Vec<(Vec, Vec)> = io_to_py_result!(res)?; + res.into_iter().map(|(k, v)| { + let k = bytes_to_string!(k)?; + let v = bytes_to_string!(v)?; + Ok((k, v)) + }).collect() + } + /// Deletes the key-value for the given key pub fn delete(&mut self, k: &str) -> PyResult<()> { io_to_py_result!(self.db.delete(k.as_bytes())) diff --git a/test/conftest.py b/test/conftest.py index ebb27b2..d0a0c2b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -22,10 +22,35 @@ ("oi", "Ronaldo"), ("mulimuta", "Aliguma"), ] +search_records = [ + ("foo", "eng"), + ("fore", "span"), + ("food", "lug"), + ("bar", "port"), + ("band", "nyoro"), + ("pig", "dan"), +] store_fixture = [lazy_fixture("sync_store")] records_fixture = [(lazy_fixture("sync_store"), k, v) for (k, v) in records] keys_fixture = [(lazy_fixture("sync_store"), k) for k in keys] +search_terms_fixture = [ + (lazy_fixture("sync_store"), term) + for term in [ + "f", + "fo", + "foo", + "for", + "b", + "ba", + "bar", + "ban", + "pigg", + "p", + "pi", + "pig", + ] +] async_store_fixture = [lazy_fixture("async_store")] @@ -44,4 +69,3 @@ async def async_store(): _store = AsyncStore(store_path=async_store_path) yield _store await _store.clear() - diff --git a/test/test_async_py_scdb.py b/test/test_async_py_scdb.py index 0e66287..d39abfc 100644 --- a/test/test_async_py_scdb.py +++ b/test/test_async_py_scdb.py @@ -4,7 +4,7 @@ import pytest from py_scdb import AsyncStore -from test.conftest import async_store_fixture, records +from test.conftest import async_store_fixture, records, search_records from test.utils import fill_async_store, get_async_db_file_size @@ -42,6 +42,28 @@ async def test_set_with_ttl(store: AsyncStore): assert (await store.get(k=k)) is None +@pytest.mark.asyncio +@pytest.mark.parametrize("store", async_store_fixture) +async def test_updating_ttl(store: AsyncStore): + """Setting a key-value again updates its ttl also""" + k1, v1 = "foo", "bar" + k2, v2 = "foo2", "bar2" + await store.set(k1, v1) # no ttl + await store.set(k2, v2, 1) # with ttl + + time.sleep(2) + assert (await store.get(k1)) == v1 + assert (await store.get(k2)) is None + + # update them, but reverse the ttl's + await store.set(k2, v2) # no ttl + await store.set(k1, v1, 1) # with ttl + + time.sleep(2) + assert (await store.get(k2)) == v2 + assert (await store.get(k1)) is None + + @pytest.mark.asyncio @pytest.mark.parametrize("store", async_store_fixture) async def test_set_existing_key(store: AsyncStore): @@ -72,6 +94,149 @@ async def test_get_non_existing_key(store: AsyncStore): assert (await store.get(k="some-random-value")) is None +@pytest.mark.asyncio +@pytest.mark.parametrize("store", async_store_fixture) +async def test_search_without_pagination(store: AsyncStore): + """Returns the list of key-values whose keys start with given search term""" + test_data = [ + ("f", [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("fo", [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("foo", [("foo", "eng"), ("food", "lug")]), + ("food", [("food", "lug")]), + ("for", [("fore", "span")]), + ("b", [("bar", "port"), ("band", "nyoro")]), + ("ba", [("bar", "port"), ("band", "nyoro")]), + ("bar", [("bar", "port")]), + ("ban", [("band", "nyoro")]), + ("band", [("band", "nyoro")]), + ("p", [("pig", "dan")]), + ("pi", [("pig", "dan")]), + ("pig", [("pig", "dan")]), + ("pigg", []), + ("bandana", []), + ("bare", []), + ] + + await fill_async_store(store=store, data=search_records) + for (term, expected) in test_data: + assert (await store.search(term=term, skip=0, limit=0)) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("store", async_store_fixture) +async def test_search_with_pagination(store: AsyncStore): + """Returns a slice of the list of key-values whose keys start with given search term, basing on skip and limit""" + test_data = [ + ("fo", 0, 0, [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("fo", 0, 8, [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("fo", 1, 8, [("fore", "span"), ("food", "lug")]), + ("fo", 1, 0, [("fore", "span"), ("food", "lug")]), + ("fo", 0, 2, [("foo", "eng"), ("fore", "span")]), + ("fo", 1, 2, [("fore", "span"), ("food", "lug")]), + ("fo", 0, 1, [("foo", "eng")]), + ("fo", 2, 1, [("food", "lug")]), + ("fo", 1, 1, [("fore", "span")]), + ] + + await fill_async_store(store=store, data=search_records) + for (term, skip, limit, expected) in test_data: + assert (await store.search(term=term, skip=skip, limit=limit)) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("store", async_store_fixture) +async def test_search_after_expiration(store: AsyncStore): + """Returns only non-expired key-values""" + records_to_expire = [search_records[0], search_records[2], search_records[3]] + await fill_async_store(store=store, data=search_records, ttl=None) + await fill_async_store(store=store, data=records_to_expire, ttl=1) + + test_data = [ + ("f", [("fore", "span")]), + ("fo", [("fore", "span")]), + ("foo", []), + ("for", [("fore", "span")]), + ("b", [("band", "nyoro")]), + ("ba", [("band", "nyoro")]), + ("bar", []), + ("ban", [("band", "nyoro")]), + ("band", [("band", "nyoro")]), + ("p", [("pig", "dan")]), + ("pi", [("pig", "dan")]), + ("pig", [("pig", "dan")]), + ("pigg", []), + ("food", []), + ("bandana", []), + ("bare", []), + ] + + # wait for some items to expire + time.sleep(2) + + for (term, expected) in test_data: + assert (await store.search(term=term, skip=0, limit=0)) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("store", async_store_fixture) +async def test_search_after_delete(store: AsyncStore): + """Returns only existing key-values""" + keys_to_delete = ["foo", "food", "bar", "band"] + await fill_async_store(store=store, data=search_records) + test_data = [ + ("f", [("fore", "span")]), + ("fo", [("fore", "span")]), + ("foo", []), + ("for", [("fore", "span")]), + ("b", []), + ("ba", []), + ("bar", []), + ("ban", []), + ("band", []), + ("p", [("pig", "dan")]), + ("pi", [("pig", "dan")]), + ("pig", [("pig", "dan")]), + ("pigg", []), + ("food", []), + ("bandana", []), + ("bare", []), + ] + + for k in keys_to_delete: + await store.delete(k) + + for (term, expected) in test_data: + assert (await store.search(term=term, skip=0, limit=0)) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("store", async_store_fixture) +async def test_search_after_clear(store: AsyncStore): + """Returns an empty list when search is called after clear""" + terms = [ + "f", + "fo", + "foo", + "for", + "b", + "ba", + "bar", + "ban", + "band", + "p", + "pi", + "pig", + "pigg", + "food", + "bandana", + "bare", + ] + await fill_async_store(store=store, data=search_records) + await store.clear() + for term in terms: + assert (await store.search(term=term, skip=0, limit=0)) == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("store", async_store_fixture) async def test_delete_existing_key(store: AsyncStore): diff --git a/test/test_benchmarks.py b/test/test_benchmarks.py index 39e7f8b..d8afabc 100644 --- a/test/test_benchmarks.py +++ b/test/test_benchmarks.py @@ -2,7 +2,14 @@ import pytest -from test.conftest import records_fixture, keys_fixture, records, store_fixture +from test.conftest import ( + records_fixture, + keys_fixture, + records, + store_fixture, + search_records, + search_terms_fixture, +) from test.utils import fill_store @@ -19,6 +26,20 @@ def test_benchmark_get(benchmark, store, k): benchmark(store.get, k=k) +@pytest.mark.parametrize("store, term", search_terms_fixture) +def test_benchmark_search(benchmark, store, term): + """Benchmarks the get operation""" + fill_store(store=store, data=search_records) + benchmark(store.search, term=term, skip=0, limit=0) + + +@pytest.mark.parametrize("store, term", search_terms_fixture) +def test_benchmark_paginated_search(benchmark, store, term): + """Benchmarks the get operation""" + fill_store(store=store, data=search_records) + benchmark(store.search, term=term, skip=1, limit=1) + + @pytest.mark.parametrize("store, k", keys_fixture) def test_benchmark_delete(benchmark, store, k): """Benchmarks the delete operation""" diff --git a/test/test_py_scdb.py b/test/test_py_scdb.py index 93549d3..d6d7b1f 100644 --- a/test/test_py_scdb.py +++ b/test/test_py_scdb.py @@ -1,10 +1,11 @@ """Tests for Store""" + import time import pytest from py_scdb import Store -from test.conftest import store_fixture, records +from test.conftest import store_fixture, records, search_records from test.utils import fill_store, get_db_file_size @@ -61,6 +62,144 @@ def test_get_existing_key(store: Store): assert store.get(k=k) == v +@pytest.mark.parametrize("store", store_fixture) +def test_search_without_pagination(store: Store): + """Returns the list of key-values whose keys start with given search term""" + test_data = [ + ("f", [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("fo", [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("foo", [("foo", "eng"), ("food", "lug")]), + ("food", [("food", "lug")]), + ("for", [("fore", "span")]), + ("b", [("bar", "port"), ("band", "nyoro")]), + ("ba", [("bar", "port"), ("band", "nyoro")]), + ("bar", [("bar", "port")]), + ("ban", [("band", "nyoro")]), + ("band", [("band", "nyoro")]), + ("p", [("pig", "dan")]), + ("pi", [("pig", "dan")]), + ("pig", [("pig", "dan")]), + ("pigg", []), + ("bandana", []), + ("bare", []), + ] + + fill_store(store=store, data=search_records) + for (term, expected) in test_data: + assert store.search(term=term, skip=0, limit=0) == expected + + +@pytest.mark.parametrize("store", store_fixture) +def test_search_with_pagination(store: Store): + """Returns a slice of the list of key-values whose keys start with given search term, basing on skip and limit""" + test_data = [ + ("fo", 0, 0, [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("fo", 0, 8, [("foo", "eng"), ("fore", "span"), ("food", "lug")]), + ("fo", 1, 8, [("fore", "span"), ("food", "lug")]), + ("fo", 1, 0, [("fore", "span"), ("food", "lug")]), + ("fo", 0, 2, [("foo", "eng"), ("fore", "span")]), + ("fo", 1, 2, [("fore", "span"), ("food", "lug")]), + ("fo", 0, 1, [("foo", "eng")]), + ("fo", 2, 1, [("food", "lug")]), + ("fo", 1, 1, [("fore", "span")]), + ] + + fill_store(store=store, data=search_records) + for (term, skip, limit, expected) in test_data: + assert store.search(term=term, skip=skip, limit=limit) == expected + + +@pytest.mark.parametrize("store", store_fixture) +def test_search_after_expiration(store: Store): + """Returns only non-expired key-values""" + records_to_expire = [search_records[0], search_records[2], search_records[3]] + fill_store(store=store, data=search_records, ttl=None) + fill_store(store=store, data=records_to_expire, ttl=1) + + test_data = [ + ("f", [("fore", "span")]), + ("fo", [("fore", "span")]), + ("foo", []), + ("for", [("fore", "span")]), + ("b", [("band", "nyoro")]), + ("ba", [("band", "nyoro")]), + ("bar", []), + ("ban", [("band", "nyoro")]), + ("band", [("band", "nyoro")]), + ("p", [("pig", "dan")]), + ("pi", [("pig", "dan")]), + ("pig", [("pig", "dan")]), + ("pigg", []), + ("food", []), + ("bandana", []), + ("bare", []), + ] + + # wait for some items to expire + time.sleep(2) + + for (term, expected) in test_data: + assert store.search(term=term, skip=0, limit=0) == expected + + +@pytest.mark.parametrize("store", store_fixture) +def test_search_after_delete(store: Store): + """Returns only existing key-values""" + keys_to_delete = ["foo", "food", "bar", "band"] + fill_store(store=store, data=search_records) + test_data = [ + ("f", [("fore", "span")]), + ("fo", [("fore", "span")]), + ("foo", []), + ("for", [("fore", "span")]), + ("b", []), + ("ba", []), + ("bar", []), + ("ban", []), + ("band", []), + ("p", [("pig", "dan")]), + ("pi", [("pig", "dan")]), + ("pig", [("pig", "dan")]), + ("pigg", []), + ("food", []), + ("bandana", []), + ("bare", []), + ] + + for k in keys_to_delete: + store.delete(k) + + for (term, expected) in test_data: + assert store.search(term=term, skip=0, limit=0) == expected + + +@pytest.mark.parametrize("store", store_fixture) +def test_search_after_clear(store: Store): + """Returns an empty list when search is called after clear""" + terms = [ + "f", + "fo", + "foo", + "for", + "b", + "ba", + "bar", + "ban", + "band", + "p", + "pi", + "pig", + "pigg", + "food", + "bandana", + "bare", + ] + fill_store(store=store, data=search_records) + store.clear() + for term in terms: + assert store.search(term=term, skip=0, limit=0) == [] + + @pytest.mark.parametrize("store", store_fixture) def test_get_non_existing_key(store: Store): """Returns the None for a key that does not exist""" diff --git a/test/utils.py b/test/utils.py index 70b4c8d..909a42b 100644 --- a/test/utils.py +++ b/test/utils.py @@ -9,7 +9,9 @@ async_store_path = path.join(_root_directory, "async_testdb") -async def fill_async_store(store: AsyncStore, data: List[Tuple[str, str]], ttl: Optional[int] = None): +async def fill_async_store( + store: AsyncStore, data: List[Tuple[str, str]], ttl: Optional[int] = None +): """Fills the async store with records""" for (key, value) in data: await store.set(k=key, v=value, ttl=ttl)