From 5eeab5a8df5622bd1937de8dc194ac8c83ad6c22 Mon Sep 17 00:00:00 2001 From: jesse Date: Fri, 19 Jul 2024 21:16:18 -0700 Subject: [PATCH] Merge many many many re-org changes into main!!! (#116) * cli with caller id' * tests for touch! * dprint update * added copy tests * bbox copy good!" * agg hash cmd * fix agg hash fn' * fix vac to be async cmd... * dbpath part of sqlite lib * lint remove dumb thing * server compression * update version * lint results * fix up touch to be async! * ack actually turn off warning if fix not not not on... should warn if true had backwards * fixes * info all async! * format * lintlintlint * lint lint lint * lint fixes for bill * pre-clippy commit with many unwraps removed * pre ci' * dev command re-org * removed panics * pragma!? * added test(s) * format * changes! * tile-size update * removed deps * fix clippy warning --- .editorconfig | 3 + .gitignore | 7 +- CHANGELOG.md | 21 +- Cargo.lock | 524 +++++------------- Cargo.toml | 36 +- README.md | 34 +- crates/utiles-core/Cargo.toml | 17 +- crates/utiles-core/src/lib.rs | 2 +- crates/utiles-core/src/tile.rs | 6 +- crates/utiles-core/src/tile_zbox.rs | 21 +- crates/utiles-dev/Cargo.toml | 26 +- crates/utiles-wasm/Cargo.toml | 3 +- crates/utiles-wasm/test/utiles-wasm.bench.ts | 54 +- crates/utiles-wasm/test/utiles-wasm.test.ts | 90 +-- crates/utiles/Cargo.toml | 64 +-- crates/utiles/src/cli/args.rs | 130 +++-- crates/utiles/src/cli/commands/about.rs | 30 +- crates/utiles/src/cli/commands/agg_hash.rs | 18 + .../src/cli/commands/children_parent.rs | 1 + crates/utiles/src/cli/commands/dev.rs | 25 +- crates/utiles/src/cli/commands/info.rs | 37 +- crates/utiles/src/cli/commands/lint.rs | 203 +------ crates/utiles/src/cli/commands/metadata.rs | 60 +- crates/utiles/src/cli/commands/mod.rs | 5 +- crates/utiles/src/cli/commands/rimraf.rs | 1 + crates/utiles/src/cli/commands/serve.rs | 18 +- crates/utiles/src/cli/commands/shapes.rs | 26 +- .../src/cli/commands/tile_stream_cmds.rs | 5 +- crates/utiles/src/cli/commands/tilejson.rs | 26 +- crates/utiles/src/cli/commands/tiles.rs | 49 +- crates/utiles/src/cli/commands/touch.rs | 64 ++- crates/utiles/src/cli/commands/update.rs | 117 ++-- crates/utiles/src/cli/commands/vacuum.rs | 25 +- crates/utiles/src/cli/commands/zxyify.rs | 2 +- crates/utiles/src/cli/entry.rs | 86 ++- crates/utiles/src/cli/mod.rs | 3 +- crates/utiles/src/cli/stdin2string.rs | 2 +- crates/utiles/src/config.rs | 39 ++ crates/utiles/src/copy/cfg.rs | 170 ++---- crates/utiles/src/copy/mod.rs | 14 +- crates/utiles/src/copy/pasta.rs | 146 +++-- crates/utiles/src/copy/pyramid.rs | 3 +- crates/utiles/src/errors.rs | 12 +- crates/utiles/src/fs_async.rs | 34 ++ crates/utiles/src/lager.rs | 1 + crates/utiles/src/lib.rs | 152 +---- crates/utiles/src/lint.rs | 263 --------- crates/utiles/src/lint/mbt_linter.rs | 134 +++++ crates/utiles/src/lint/mod.rs | 140 +++++ crates/utiles/src/mbt/agg_tiles_hash.rs | 95 ++-- crates/utiles/src/mbt/hash_types.rs | 7 +- crates/utiles/src/mbt/mbt_stats.rs | 74 ++- crates/utiles/src/mbt/metadata/change.rs | 58 +- crates/utiles/src/mbt/metadata/mod.rs | 3 +- crates/utiles/src/mbt/metadata/read_fspath.rs | 2 +- crates/utiles/src/mbt/metadata_row.rs | 134 ++++- crates/utiles/src/mbt/mod.rs | 4 +- crates/utiles/src/mbt/query.rs | 229 ++++---- .../mbt/sql/mbt-metadata-duplicates-json.sql | 35 ++ crates/utiles/src/mbt/tiles_filter.rs | 73 +++ crates/utiles/src/server/mod.rs | 30 +- crates/utiles/src/sqlite/async_sqlite3.rs | 112 +++- crates/utiles/src/sqlite/db.rs | 51 +- .../src/{utilesqlite => sqlite}/dbpath.rs | 31 +- crates/utiles/src/sqlite/errors.rs | 3 + crates/utiles/src/sqlite/insert_strategy.rs | 4 +- crates/utiles/src/sqlite/mod.rs | 6 +- crates/utiles/src/sqlite/page_size.rs | 19 - crates/utiles/src/sqlite/pragma.rs | 128 ++++- crates/utiles/src/sqlite/sqlike3.rs | 6 +- crates/utiles/src/sqlite_utiles/hash_int.rs | 47 +- crates/utiles/src/sqlite_utiles/mod.rs | 21 +- crates/utiles/src/tests/core.rs | 109 ++++ crates/utiles/src/tests/mod.rs | 2 + crates/utiles/src/utilejson.rs | 217 ++++++++ crates/utiles/src/utilesqlite/mbtiles.rs | 139 ++--- .../utiles/src/utilesqlite/mbtiles_async.rs | 6 +- .../src/utilesqlite/mbtiles_async_sqlite.rs | 221 +++----- crates/utiles/src/utilesqlite/mod.rs | 1 - dprint.json | 37 ++ justfile | 10 +- utiles-pyo3/Cargo.toml | 8 +- utiles-pyo3/README.md | 9 +- utiles-pyo3/pyproject.toml | 286 ++++------ utiles-pyo3/python/utiles/dev/testing.py | 15 +- utiles-pyo3/requirements/dev.in | 1 + utiles-pyo3/requirements/dev.txt | 13 +- utiles-pyo3/src/cli.rs | 8 +- utiles-pyo3/src/pyutiles/pytile.rs | 124 ++--- utiles-pyo3/tests/cli/test_copy_db.py | 211 +++++++ .../{test_copy.py => test_copy_pyramid.py} | 0 utiles-pyo3/tests/cli/test_db.py | 23 +- utiles-pyo3/tests/cli/test_mt.py | 2 +- utiles-pyo3/tests/cli/test_update.py | 52 ++ utiles-pyo3/tests/conftest.py | 6 + 95 files changed, 3150 insertions(+), 2471 deletions(-) create mode 100644 crates/utiles/src/cli/commands/agg_hash.rs create mode 100644 crates/utiles/src/config.rs create mode 100644 crates/utiles/src/fs_async.rs delete mode 100644 crates/utiles/src/lint.rs create mode 100644 crates/utiles/src/lint/mbt_linter.rs create mode 100644 crates/utiles/src/lint/mod.rs create mode 100644 crates/utiles/src/mbt/sql/mbt-metadata-duplicates-json.sql create mode 100644 crates/utiles/src/mbt/tiles_filter.rs rename crates/utiles/src/{utilesqlite => sqlite}/dbpath.rs (80%) create mode 100644 crates/utiles/src/tests/core.rs create mode 100644 crates/utiles/src/tests/mod.rs create mode 100644 dprint.json create mode 100644 utiles-pyo3/tests/cli/test_copy_db.py rename utiles-pyo3/tests/cli/{test_copy.py => test_copy_pyramid.py} (100%) create mode 100644 utiles-pyo3/tests/cli/test_update.py diff --git a/.editorconfig b/.editorconfig index 64f64128..d122e146 100644 --- a/.editorconfig +++ b/.editorconfig @@ -44,6 +44,9 @@ max_line_length = 80 indent_size = 2 indent_style = space +[*.{toml}] +indent_size = 2 + [*.{yml,yaml}] indent_style = space indent_size = 2 diff --git a/.gitignore b/.gitignore index 135295b4..7d2a5ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ .venv/ .tox/ playground/ +__pycache__/ f.py f.ts f.tsx @@ -175,9 +176,6 @@ cdk.out/ .vscode/ .vscode/* # Maybe .vscode/**/* instead - see comments !.vscode/settings.json.default -# !.vscode/tasks.json -# !.vscode/launch.json -# !.vscode/extensions.json ### VisualStudioCode Patch ### # Ignore all local history of files @@ -195,5 +193,8 @@ file.py # testing data blue-marble/ +crates/**/*.db crates/**/*.mbtiles +crates/**/*.pmtiles +crates/**/*.sqlite crates/**/*.utiles diff --git a/CHANGELOG.md b/CHANGELOG.md index 2390a6d5..d1ad613a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # CHANGELOG ## TODO - + - lint/copy overhaul - `webpify` command for converting images to webp @@ -12,14 +12,13 @@ - Using `json-patch` for metadata updates - Allow setting metadata value(s) from file if no value is provided (`-`/`--`) for stdin -___ - +--- ## 0.6.1 (2024-07-01) - Fix calling `utiles.ut_cli` multiple times causing tracing-subscriber crash -___ +--- ## 0.6.0 (2024-06-28) @@ -27,7 +26,7 @@ ___ - Update python dev deps - Added `{bbox}`, `{projwin}`, `{bbox_web}` and `{projwin_web}` format tokens to tile-formatter (those projwins are handy for gdaling) -___ +--- ## 0.5.1 (2024-06-19) @@ -35,7 +34,7 @@ ___ - Write out `metadata.json` when `pyramid-ing` mbtiles to directory if the metadata of the mbtiles does not conatin duplicate keys (which it should not) - Limit jobs/concurrency when `pyramid-ing` mbtiles to directory to 4 (if not specified by `--jobs`/`-j` option) to prevent nuking machines -___ +--- ## 0.5.0 (2024-06-14) @@ -77,7 +76,7 @@ Example: SELECT * FROM tiles WHERE zoom_level = 10 AND tile_column = 486 AND tile_row = 332; ``` -___ +--- ## 0.4.1 (2024-04-04) @@ -90,7 +89,7 @@ ___ - General spring cleaning! - Hid the `utiles tilejson` cli alias `trader-joes` -___ +--- ## 0.3.1 (2024-01-30) @@ -100,7 +99,7 @@ ___ - Expanded utiles cli with several more commands -___ +--- ## 0.2.0 (2023-11-10) @@ -109,7 +108,7 @@ ___ - Added tilejson/tj command to rust cli to write out tilejson files for mbtiles - Added meta command to rust cli to write out json of metadata table for mbtiles -___ +--- ## 0.1.0 (2023-10-27) @@ -117,7 +116,7 @@ ___ - Update pyo3 to 0.20.0 - Added rasterio/rio entry points ('utiles' and 'ut' alias bc why type `rio utiles` over `rio ut`) -___ +--- ## 0.0.2 diff --git a/Cargo.lock b/Cargo.lock index 04e1ce16..65e04118 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", "version_check", "zerocopy", @@ -117,6 +117,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-sqlite" version = "0.3.0" @@ -137,7 +152,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -155,7 +170,7 @@ dependencies = [ "async-trait", "axum-core", "axum-macros", - "bytes 1.6.0", + "bytes", "futures-util", "http", "http-body", @@ -188,7 +203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" dependencies = [ "async-trait", - "bytes 1.6.0", + "bytes", "futures-util", "http", "http-body", @@ -211,7 +226,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -222,7 +237,7 @@ checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide", "object", @@ -266,27 +281,11 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" -version = "0.4.12" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - -[[package]] -name = "bytes" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cast" @@ -296,15 +295,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "066fce287b1d4eafef758e89e09d724a24808a9196fe9756b8ca90e86d0719a2" - -[[package]] -name = "cfg-if" -version = "0.1.10" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +checksum = "324c74f2155653c90b04f25b2a47a8a631360cb908f92a772695f430c7e31052" +dependencies = [ + "jobserver", + "libc", +] [[package]] name = "cfg-if" @@ -385,7 +382,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -394,15 +391,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "colorchoice" version = "1.0.1" @@ -425,6 +413,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -467,7 +464,7 @@ version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ - "crossbeam-utils 0.8.20", + "crossbeam-utils", ] [[package]] @@ -477,7 +474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ "crossbeam-epoch", - "crossbeam-utils 0.8.20", + "crossbeam-utils", ] [[package]] @@ -486,18 +483,7 @@ version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "crossbeam-utils 0.8.20", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "crossbeam-utils", ] [[package]] @@ -584,6 +570,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flate2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fluent-uri" version = "0.1.4" @@ -608,28 +604,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags 1.3.2", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.30" @@ -686,7 +660,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -759,7 +733,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi", ] @@ -789,7 +763,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crunchy", ] @@ -841,18 +815,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.6.0", + "bytes", "fnv", "itoa", ] [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes 1.6.0", + "bytes", "http", ] @@ -862,7 +836,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-util", "http", "http-body", @@ -883,11 +857,11 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-channel", "futures-util", "http", @@ -896,7 +870,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "smallvec 1.13.2", + "smallvec", "tokio", ] @@ -906,7 +880,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-util", "http", "http-body", @@ -960,15 +934,6 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "is-terminal" version = "0.4.12" @@ -1001,6 +966,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.69" @@ -1033,16 +1007,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -1078,15 +1042,6 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" -[[package]] -name = "lock_api" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -dependencies = [ - "scopeguard", -] - [[package]] name = "lock_api" version = "0.4.12" @@ -1118,19 +1073,13 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md-5" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "digest", ] @@ -1164,25 +1113,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.8.11" @@ -1194,40 +1124,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - -[[package]] -name = "net2" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "noncrypto-digests" version = "0.3.2" @@ -1246,7 +1142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1302,40 +1198,14 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.6.3", - "rustc_version", -] - [[package]] name = "parking_lot" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ - "lock_api 0.4.12", - "parking_lot_core 0.9.10", -] - -[[package]] -name = "parking_lot_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "rustc_version", - "smallvec 0.6.14", - "winapi 0.3.9", + "lock_api", + "parking_lot_core", ] [[package]] @@ -1344,10 +1214,10 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", - "redox_syscall 0.5.2", - "smallvec 1.13.2", + "redox_syscall", + "smallvec", "windows-targets 0.52.6", ] @@ -1374,7 +1244,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1450,7 +1320,7 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e99090d12f6182924499253aaa1e73bf15c69cea8d2774c3c781e35badc3548" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "indoc", "libc", "memoffset", @@ -1491,7 +1361,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1504,12 +1374,12 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] name = "pyutiles" -version = "0.6.1" +version = "0.7.0-alpha.2" dependencies = [ "pyo3", "pyo3-build-config", @@ -1546,20 +1416,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", - "crossbeam-utils 0.8.20", + "crossbeam-utils", ] [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - -[[package]] -name = "redox_syscall" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ "bitflags 2.6.0", ] @@ -1619,7 +1483,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "smallvec 1.13.2", + "smallvec", ] [[package]] @@ -1628,15 +1492,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.38.34" @@ -1677,21 +1532,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" version = "1.0.204" @@ -1709,7 +1549,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1800,15 +1640,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -1863,7 +1694,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1879,9 +1710,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.70" +version = "2.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" dependencies = [ "proc-macro2", "quote", @@ -1933,7 +1764,7 @@ checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1942,7 +1773,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -1994,11 +1825,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", - "bytes 1.6.0", + "bytes", "libc", - "mio 0.8.11", + "mio", "num_cpus", - "parking_lot 0.12.3", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2006,27 +1837,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-executor" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", -] - -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-macros" version = "2.3.0" @@ -2035,66 +1845,22 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] -name = "tokio-reactor" -version = "0.1.12" +name = "tokio-util" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "mio 0.6.23", - "num_cpus", - "parking_lot 0.9.0", - "slab", - "tokio-executor", - "tokio-io", - "tokio-sync", -] - -[[package]] -name = "tokio-signal" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12" -dependencies = [ - "futures 0.1.31", - "libc", - "mio 0.6.23", - "mio-uds", - "signal-hook-registry", - "tokio-executor", - "tokio-io", - "tokio-reactor", - "winapi 0.3.9", -] - -[[package]] -name = "tokio-stream" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ + "bytes", "futures-core", + "futures-sink", "pin-project-lite", "tokio", ] -[[package]] -name = "tokio-sync" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" -dependencies = [ - "fnv", - "futures 0.1.31", -] - [[package]] name = "tower" version = "0.4.13" @@ -2117,13 +1883,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ + "async-compression", "bitflags 2.6.0", - "bytes 1.6.0", + "bytes", + "futures-core", "http", "http-body", "http-body-util", "pin-project-lite", "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -2163,7 +1932,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2211,7 +1980,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec 1.13.2", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -2245,7 +2014,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utiles" -version = "0.6.1" +version = "0.7.0-alpha.2" dependencies = [ "async-sqlite", "async-trait", @@ -2254,15 +2023,14 @@ dependencies = [ "clap", "colored", "criterion", - "fast_hilbert", - "futures 0.3.30", + "fnv", + "futures", "geo-types", "geojson", "globset", - "http-body-util", "imagesize", + "indoc", "json-patch", - "log", "num_cpus", "rusqlite", "serde", @@ -2274,8 +2042,6 @@ dependencies = [ "tilejson", "time", "tokio", - "tokio-signal", - "tokio-stream", "tower", "tower-http", "tracing", @@ -2287,7 +2053,7 @@ dependencies = [ [[package]] name = "utiles-core" -version = "0.6.1" +version = "0.7.0-alpha.2" dependencies = [ "fast_hilbert", "serde", @@ -2344,7 +2110,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -2359,7 +2125,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "wasm-bindgen-shared", ] @@ -2381,7 +2147,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2402,12 +2168,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -2418,12 +2178,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -2593,16 +2347,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "xxhash-rust" version = "0.8.11" @@ -2626,5 +2370,33 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.12+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +dependencies = [ + "cc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index c34c8f84..92bf9187 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,43 +1,45 @@ [workspace] resolver = "2" members = [ - "crates/utiles-core", - "crates/utiles", - "utiles-pyo3", + "crates/utiles", + "crates/utiles-core", + "utiles-pyo3", ] [workspace.package] +version = "0.7.0-alpha.2" +authors = [ + "Jesse Rubin ", + "Dan Costello ", +] +documentation = "https://github.com/jessekrubin/utiles" edition = "2021" -version = "0.6.1" homepage = "https://github.com/jessekrubin/utiles" -documentation = "https://github.com/jessekrubin/utiles" -repository = "https://github.com/jessekrubin/utiles" -authors = ["Jesse K. Rubin "] license = "MIT OR Apache-2.0" +repository = "https://github.com/jessekrubin/utiles" [workspace.dependencies] anyhow = "1.0.75" +# GIT DEP +async-sqlite = { version = "0.3.0", features = ["bundled", "functions", "trace"] } fast_hilbert = "2.0.0" geo-types = "0.7.9" geojson = "0.24.1" +indoc = "2.0.5" pyo3 = "0.22.0" pyo3-build-config = "0.22.0" rusqlite = { version = "0.31.0", features = ["bundled", "vtab", "blob"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.119", features = ["preserve_order"] } - +size = { version = "=0.5.0-preview2", features = ["default"] } +strum = { version = "0.26.3", features = ["derive"] } +strum_macros = "0.26.3" thiserror = "1.0.62" tilejson = "0.4.1" tokio = { version = "1.37.0", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.17", features = ["serde", "serde_json", "env-filter"] } -size = { version = "=0.5.0-preview2", features = ["default"] } -strum = { version = "0.26.3", features = ["derive"] } -strum_macros = "0.26.3" -xxhash-rust = { version = "0.8.10", features = [ "xxh32", "xxh64", "xxh3", "const_xxh32", "const_xxh64", "const_xxh3"] } -# GIT DEP -async-sqlite = { version = "0.3.0", features = ["bundled", "functions", "trace", ] } - +xxhash-rust = { version = "0.8.10", features = ["xxh32", "xxh64", "xxh3", "const_xxh32", "const_xxh64", "const_xxh3"] } [profile.dev] opt-level = 0 @@ -45,5 +47,5 @@ opt-level = 0 [profile.release] opt-level = 3 strip = true -#lto = "thin" -lto = "fat" +lto = "thin" +# lto = "fat" diff --git a/README.md b/README.md index e34331e8..31203f28 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ utiles = utils + tiles - ## Installation ```bash @@ -29,7 +28,7 @@ a less slim crate with a lib/cli (`utiles`), and the python wrapper package. For more details on the python package see: [./utiles-pyo3](https://github.com/jessekrubin/utiles/tree/main/utiles-pyo3) -### Why? +### Why? I use mercantile regularly and wished it were a bit more ergonomic, had type annotations, and was faster, but overall it's a great library. @@ -47,34 +46,34 @@ Not quite, but it's close. utiles doesn't throw the same exceptions as mercantil There might be other differences, but I have been using it instead of mercantile for a bit now and it works pretty decent, tho I am open to suggestions! -___ +--- # dev -## Contributing +## Contributing - - Please do! Would love some feedback! - - Be kind! - - I will happily accept PRs, and add you to the currently (5/26/2023) non-existent contributors list. +- Please do! Would love some feedback! +- Be kind! +- I will happily accept PRs, and add you to the currently (5/26/2023) non-existent contributors list. ## TODO: - - - [X] benchmark against mercantile - - **Maybe:** - - [X] Split library into `utiles` (rust lib) and `utiles-python` (python/pip package)? - - [] Mbtiles support?? - - [] Reading/writing mvt files? - - [] Re-write cli in rust with clap? -___ +- [x] benchmark against mercantile +- **Maybe:** + - [x] Split library into `utiles` (rust lib) and `utiles-python` (python/pip package)? + - [] Mbtiles support?? + - [] Reading/writing mvt files? + - [] Re-write cli in rust with clap? + +--- -## MISC +## MISC
zoom info | zoom | ntiles | total | rowcol_range | max_rowcol | -|-----:|--------------------------:|--------------------------:|--------------:|--------------:| +| ---: | ------------------------: | ------------------------: | ------------: | ------------: | | 0 | 1 | 1 | 0 | 1 | | 1 | 4 | 5 | 1 | 2 | | 2 | 16 | 21 | 3 | 4 | @@ -110,7 +109,6 @@ ___
- Zoom levels ``` diff --git a/crates/utiles-core/Cargo.toml b/crates/utiles-core/Cargo.toml index 302c7786..c43c211a 100644 --- a/crates/utiles-core/Cargo.toml +++ b/crates/utiles-core/Cargo.toml @@ -1,20 +1,19 @@ [package] name = "utiles-core" version.workspace = true -edition.workspace = true -license.workspace = true authors.workspace = true -description = "Map tile utilities aka utiles" -readme = "README.md" categories = ["science::geo"] -keywords = ["map", "geo", "mercator", "tile"] +edition.workspace = true homepage = "https://github.com/jessekrubin/utiles" +keywords = ["map", "geo", "mercator", "tile"] +license.workspace = true +readme = "README.md" repository = "https://github.com/jessekrubin/utiles" +description = "Map tile utilities aka utiles" [dependencies] -thiserror.workspace = true -serde.workspace = true -serde_json.workspace = true # Not going to get rid of fast_hilbert fast_hilbert.workspace = true - +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true diff --git a/crates/utiles-core/src/lib.rs b/crates/utiles-core/src/lib.rs index 9e903e0c..a67a070b 100644 --- a/crates/utiles-core/src/lib.rs +++ b/crates/utiles-core/src/lib.rs @@ -64,7 +64,7 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); /// Tile macro to create a new tile. /// - do you need this? probably not -/// - Did I write to to figure out how to write a macro? yes +/// - Did I write to figure out how to write a macro? yes #[macro_export] macro_rules! utile { ($x:expr, $y:expr, $z:expr) => { diff --git a/crates/utiles-core/src/tile.rs b/crates/utiles-core/src/tile.rs index a4d4337e..2a44694f 100644 --- a/crates/utiles-core/src/tile.rs +++ b/crates/utiles-core/src/tile.rs @@ -11,7 +11,6 @@ use crate::constants::EPSILON; use crate::errors::UtilesCoreError; use crate::errors::UtilesCoreResult; use crate::fns::{bounds, children, neighbors, parent, siblings, xy}; -// use crate::mbutiles::MbtTileRow; use crate::projection::Projection; use crate::tile_feature::TileFeature; use crate::tile_like::TileLike; @@ -484,10 +483,7 @@ impl Tile { } /// Return a `TileFeature` for the tile - pub fn feature( - &self, - opts: &FeatureOptions, - ) -> Result> { + pub fn feature(&self, opts: &FeatureOptions) -> UtilesCoreResult { let buffer = opts.buffer.unwrap_or(0.0); let precision = opts.precision.unwrap_or(-1); // Compute the bounds diff --git a/crates/utiles-core/src/tile_zbox.rs b/crates/utiles-core/src/tile_zbox.rs index f15fe0a4..9882f28c 100644 --- a/crates/utiles-core/src/tile_zbox.rs +++ b/crates/utiles-core/src/tile_zbox.rs @@ -102,20 +102,27 @@ impl TileZBox { && tile.y() <= self.max.y } - /// Return the SQL `WHERE` clause for an mbtiles database + /// Return the SQL `WHERE` clause for tms mbtiles like db with optional prefix for column names #[must_use] - pub fn mbtiles_sql_where(&self) -> String { + pub fn mbtiles_sql_where_prefix(&self, prefix: Option<&str>) -> String { + let col_prefix = prefix.unwrap_or_default(); // classic mbtiles sqlite query: // 'SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?', let miny = crate::fns::flipy(self.min.y, self.zoom); let maxy = crate::fns::flipy(self.max.y, self.zoom); format!( - "(zoom_level = {} AND tile_column >= {} AND tile_column <= {} AND tile_row >= {} AND tile_row <= {})", + "(zoom_level = {} AND {}tile_column >= {} AND {}tile_column <= {} AND {}tile_row >= {} AND {}tile_row <= {})", self.zoom, - self.min.x, self.max.x, - maxy, miny + col_prefix, self.min.x, col_prefix, self.max.x, + col_prefix, maxy, col_prefix, miny ) } + + /// Return the SQL `WHERE` clause for an mbtiles database + #[must_use] + pub fn mbtiles_sql_where(&self) -> String { + self.mbtiles_sql_where_prefix(None) + } } impl TileZBoxIterator { @@ -212,10 +219,10 @@ impl TileZBoxes { /// Return the size of the `TileZBoxes` in tiles #[must_use] - pub fn mbtiles_sql_where(&self) -> String { + pub fn mbtiles_sql_where(&self, prefix: Option<&str>) -> String { self.ranges .iter() - .map(TileZBox::mbtiles_sql_where) + .map(move |r| r.mbtiles_sql_where_prefix(prefix)) .collect::>() .join(" OR ") } diff --git a/crates/utiles-dev/Cargo.toml b/crates/utiles-dev/Cargo.toml index 526c611f..18f90dd4 100644 --- a/crates/utiles-dev/Cargo.toml +++ b/crates/utiles-dev/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "utiles-dev" -edition.workspace = true version.workspace = true -homepage.workspace = true -documentation.workspace = true -repository.workspace = true authors.workspace = true +documentation.workspace = true +edition.workspace = true +homepage.workspace = true license.workspace = true +repository.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] @@ -18,22 +18,22 @@ name = "utdev" path = "src/main.rs" [dependencies] -utiles = { path = "../utiles" } -utilesqlite = { path = "../utilesqlite" } -utiles-cli = { path = "../utiles-cli" } anyhow.workspace = true +deadpool-sqlite = { version = "0.7.0", features = ["tracing"] } fast_hilbert.workspace = true +futures = "0.3.29" geo-types.workspace = true geojson.workspace = true +geozero = { version = "0.11.0", features = ["with-mvt", "with-wkb"] } rusqlite.workspace = true serde.workspace = true serde_json.workspace = true -thiserror.workspace= true +sqlx = { version = "0.7.2", features = ["sqlite", "runtime-tokio", "macros"] } +thiserror.workspace = true tilejson.workspace = true tokio.workspace = true -tracing-subscriber.workspace = true tracing.workspace = true -sqlx = { version = "0.7.2", features = ["sqlite", "runtime-tokio", "macros"] } -futures = "0.3.29" -geozero = { version = "0.11.0", features = ["with-mvt", "with-wkb"] } -deadpool-sqlite = { version = "0.7.0", features = ["tracing"] } +tracing-subscriber.workspace = true +utiles = { path = "../utiles" } +utiles-cli = { path = "../utiles-cli" } +utilesqlite = { path = "../utilesqlite" } diff --git a/crates/utiles-wasm/Cargo.toml b/crates/utiles-wasm/Cargo.toml index 83a0e2c8..9d5f566d 100644 --- a/crates/utiles-wasm/Cargo.toml +++ b/crates/utiles-wasm/Cargo.toml @@ -6,10 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +utiles = { path = "../utiles" } wasm-bindgen = "0.2.88" wasm-bindgen-test = "0.3.38" -utiles = { path = "../utiles" } [lib] crate-type = ["cdylib"] - diff --git a/crates/utiles-wasm/test/utiles-wasm.bench.ts b/crates/utiles-wasm/test/utiles-wasm.bench.ts index 05895ab3..37134f1a 100644 --- a/crates/utiles-wasm/test/utiles-wasm.bench.ts +++ b/crates/utiles-wasm/test/utiles-wasm.bench.ts @@ -1,38 +1,40 @@ -import {test, expect, bench} from 'vitest' -import * as utw from '../pkg/utiles_wasm.js' -import * as pmtiles from 'pmtiles' +import * as pmtiles from "pmtiles"; +import { bench, expect, test } from "vitest"; +import * as utw from "../pkg/utiles_wasm.js"; function xyz2quadkey( - x: number, y: number, z: number + x: number, + y: number, + z: number, ) { - let quadkey = '' + let quadkey = ""; for (let i = z; i > 0; i--) { - let digit = 0 - const mask = 1 << (i - 1) - if ((x & mask) !== 0) digit += 1 - if ((y & mask) !== 0) digit += 2 - quadkey += digit + let digit = 0; + const mask = 1 << (i - 1); + if ((x & mask) !== 0) digit += 1; + if ((y & mask) !== 0) digit += 2; + quadkey += digit; } - return quadkey + return quadkey; } -bench('js-quadkey', () => { - xyz2quadkey(486, 332, 20) -}) +bench("js-quadkey", () => { + xyz2quadkey(486, 332, 20); +}); -bench('wasm-quadkey', () => { - utw.xyz2qk(486, 332, 20) -}) +bench("wasm-quadkey", () => { + utw.xyz2qk(486, 332, 20); +}); - -bench('js-pmtile-id', () => { +bench("js-pmtile-id", () => { pmtiles.zxyToTileId( - 20, 486, 332 - ) + 20, + 486, + 332, + ); // .pmtileId2(486, 332, 20) - }) - +}); -bench('wasm-pmtile-id', () => { - utw.pmtileid(486, 332, 20) -}) +bench("wasm-pmtile-id", () => { + utw.pmtileid(486, 332, 20); +}); diff --git a/crates/utiles-wasm/test/utiles-wasm.test.ts b/crates/utiles-wasm/test/utiles-wasm.test.ts index 7c24df7b..df5c280a 100644 --- a/crates/utiles-wasm/test/utiles-wasm.test.ts +++ b/crates/utiles-wasm/test/utiles-wasm.test.ts @@ -1,72 +1,78 @@ -import {test, expect, bench} from 'vitest' -import * as utw from '../pkg/utiles_wasm.js' -import * as pmtiles from 'pmtiles' +import * as pmtiles from "pmtiles"; +import { bench, expect, test } from "vitest"; +import * as utw from "../pkg/utiles_wasm.js"; function xyz2quadkey( - x: number, y: number, z: number + x: number, + y: number, + z: number, ) { - let quadkey = '' + let quadkey = ""; for (let i = z; i > 0; i--) { - let digit = 0 - const mask = 1 << (i - 1) - if ((x & mask) !== 0) digit += 1 - if ((y & mask) !== 0) digit += 2 - quadkey += digit + let digit = 0; + const mask = 1 << (i - 1); + if ((x & mask) !== 0) digit += 1; + if ((y & mask) !== 0) digit += 2; + quadkey += digit; } - return quadkey + return quadkey; } test( - 'uno', () => { - console.log('uno') - expect(1).toBe(1) - } -) + "uno", + () => { + console.log("uno"); + expect(1).toBe(1); + }, +); // 486, 332, 20 test( - 'thingy', async () => { - console.log('thingy') - console.log(utw) - const added = utw.add(1, 2) - expect(added).toBe(3) - + "thingy", + async () => { + console.log("thingy"); + console.log(utw); + const added = utw.add(1, 2); + expect(added).toBe(3); // const called = await utw.default() // console.log(called) // expect(utw.add(1, 2)).toBe(3) - } -) + }, +); test( - 'quadkey', async () => { - const jsres = xyz2quadkey(486, 332, 20) - const wasmres = utw.xyz2qk(486, 332, 20) - expect(jsres).toBe(wasmres) - + "quadkey", + async () => { + const jsres = xyz2quadkey(486, 332, 20); + const wasmres = utw.xyz2qk(486, 332, 20); + expect(jsres).toBe(wasmres); // const called = await utw.default() // console.log(called) // expect(utw.add(1, 2)).toBe(3) - } -) + }, +); test( - 'pmtileid', async () => { + "pmtileid", + async () => { const jsres = pmtiles.zxyToTileId( - 15, 486, 332 - ) - const wasmres = utw.pmtileid(486, 332, 15) + 15, + 486, + 332, + ); + const wasmres = utw.pmtileid(486, 332, 15); console.log( { jsres, - wasmres - } - ) + wasmres, + }, + ); expect(jsres).toBe( // debigint - Number(wasmres) - - ) - }) + Number(wasmres), + ); + }, +); diff --git a/crates/utiles/Cargo.toml b/crates/utiles/Cargo.toml index 2d11ea77..db24794d 100644 --- a/crates/utiles/Cargo.toml +++ b/crates/utiles/Cargo.toml @@ -1,15 +1,15 @@ [package] name = "utiles" version.workspace = true -edition.workspace = true -license.workspace = true authors.workspace = true -description = "Map tile utilities aka utiles" -readme = "README.md" categories = ["science::geo"] -keywords = ["map", "geo", "mercator", "tile"] +edition.workspace = true homepage = "https://github.com/jessekrubin/utiles" +keywords = ["map", "geo", "mercator", "tile"] +license.workspace = true +readme = "README.md" repository = "https://github.com/jessekrubin/utiles" +description = "Web map tile utils (aka utiles)" [lib] name = "utiles" @@ -20,46 +20,46 @@ name = "utiles" path = "src/bin.rs" [dependencies] -utiles-core = { path = "../utiles-core", version = "0.6.0" } -fast_hilbert.workspace = true +# fast_hilbert.workspace = true geo-types.workspace = true geojson.workspace = true serde.workspace = true -serde_json= { workspace = true, features = ["preserve_order"] } -thiserror.workspace = true -tilejson.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } strum.workspace = true strum_macros.workspace = true -xxhash-rust = {workspace = true , features = ["const_xxh3", "const_xxh64", "const_xxh32", "xxh3", "xxh64", "xxh32" ]} +thiserror.workspace = true +tilejson.workspace = true +utiles-core = { path = "../utiles-core", version = "0.7.0-alpha.1" } +xxhash-rust = { workspace = true, features = ["const_xxh3", "const_xxh64", "const_xxh32", "xxh3", "xxh64", "xxh32"] } # CLI dependencies -tracing.workspace = true -tokio = { workspace = true, features = ["fs"] } -tracing-subscriber = { workspace = true, features = ["fmt", "json", "env-filter", "chrono"] } +# async-sqlite = { version = "0.2.2", features = ["bundled", "functions", "trace", ] } +async-sqlite = { workspace = true, features = ["bundled", "functions", "trace", "blob"] } +async-trait = "0.1.80" +axum = { version = "0.7.5", features = ["tokio", "json", "macros"] } +chrono = "0.4.38" clap = { version = "4.5.8", features = ["derive", "color", "wrap_help"] } -globset = "0.4.13" -tokio-stream = "0.1.14" +colored = "2.1.0" futures = "0.3.29" -walkdir = "2.4.0" -time = "0.3.36" +globset = "0.4.13" +imagesize = "0.13.0" +indoc = { workspace = true } +json-patch = "2.0.0" +num_cpus = "1.16.0" # utilesqlite dependencies rusqlite = { workspace = true, features = ["bundled", "blob", "backup", "functions", "trace"] } sqlite-hashes = { version = "0.7.3", default-features = false, features = ["hex", "window", "md5", "fnv", "xxhash"] } -imagesize = "0.13.0" -axum = { version = "0.7.5", features = ["tokio", "json", "macros"] } -http-body-util = "0.1.0" -async-trait = "0.1.80" -#async-sqlite = { version = "0.2.2", features = ["bundled", "functions", "trace", ] } -async-sqlite = { workspace = true, features = ["bundled", "functions", "trace", "blob"] } +time = "0.3.36" +tokio = { workspace = true, features = ["fs"] } +# tokio-signal = "0.2.9" +# tokio-stream = "0.1.14" tower = { version = "0.4.13", features = ["timeout"] } -tower-http = { version = "0.5.1", features = ["trace", "timeout", "add-extension", "util", "request-id"] } -tokio-signal = "0.2.9" -colored = "2.1.0" -chrono = "0.4.38" -num_cpus = "1.16.0" -json-patch = "2.0.0" -log = "0.4.22" -#image = "0.25.1" +tower-http = { version = "0.5.1", features = ["trace", "timeout", "add-extension", "util", "request-id", "compression-gzip", "compression-zstd", "async-compression"] } +tracing.workspace = true +tracing-subscriber = { workspace = true, features = ["fmt", "json", "env-filter", "chrono"] } +walkdir = "2.4.0" +fnv = "1.0.7" +# image = "0.25.1" [dev-dependencies] criterion = "0.5.1" diff --git a/crates/utiles/src/cli/args.rs b/crates/utiles/src/cli/args.rs index dd01e869..4c479eee 100644 --- a/crates/utiles/src/cli/args.rs +++ b/crates/utiles/src/cli/args.rs @@ -1,25 +1,20 @@ -use std::path::PathBuf; - use clap::{Args, Parser, Subcommand}; use strum_macros::AsRefStr; -use utiles_core::bbox::BBox; -use utiles_core::parsing::parse_bbox_ext; -use utiles_core::zoom::ZoomSet; -use utiles_core::LngLat; -use utiles_core::VERSION; -use utiles_core::{geobbox_merge, zoom}; +use utiles_core::{ + geobbox_merge, parsing::parse_bbox_ext, zoom, BBox, LngLat, ZoomSet, VERSION, +}; use crate::cli::commands::dev::DevArgs; use crate::cli::commands::serve::ServeArgs; use crate::cli::commands::shapes::ShapesArgs; -use crate::copy::CopyConfig; -use crate::mbt::MbtType; +use crate::cli::commands::{analyze_main, vacuum_main}; +use crate::errors::UtilesResult; +use crate::mbt::hash_types::HashType; +use crate::mbt::{MbtType, TilesFilter}; use crate::sqlite::InsertStrategy; use crate::tile_strfmt::TileStringFormatter; -// use crate::cli::commands::WebpifyArgs; - /// ██╗ ██╗████████╗██╗██╗ ███████╗███████╗ /// ██║ ██║╚══██╔══╝██║██║ ██╔════╝██╔════╝ /// ██║ ██║ ██║ ██║██║ █████╗ ███████╗ @@ -183,6 +178,40 @@ pub struct SqliteDbCommonArgs { pub min: bool, } +#[derive(Debug, Parser)] +pub struct TilesFilterArgs { + /// bbox(es) (west, south, east, north) + #[arg(required = false, long, value_parser = parse_bbox_ext, allow_hyphen_values = true)] + pub bbox: Option>, + + #[command(flatten)] + pub zoom: Option, +} + +impl TilesFilterArgs { + #[must_use] + pub fn zooms(&self) -> Option> { + match &self.zoom { + Some(zoom) => zoom.zooms(), + None => None, + } + } + + #[must_use] + pub fn bboxes(&self) -> Option> { + self.bbox.clone() + } + + #[must_use] + pub fn tiles_filter_maybe(&self) -> Option { + if self.bbox.is_none() && self.zoom.is_none() { + None + } else { + Some(TilesFilter::new(self.bboxes(), self.zooms())) + } + } +} + #[derive(Debug, Parser, Clone, clap::ValueEnum)] pub enum DbtypeOption { Flat, @@ -211,7 +240,9 @@ pub struct TouchArgs { pub page_size: Option, /// db-type (default: flat) - #[arg(required = false, long = "dbtype", default_value = "flat")] + #[arg( + required = false, long = "dbtype", aliases = ["db-type", "mbtype", "mbt-type"], default_value = "flat" + )] pub dbtype: Option, } @@ -222,6 +253,22 @@ impl TouchArgs { } } +#[derive(Debug, Subcommand)] +/// sqlite utils/cmds +pub enum DbCommands { + Analyze(AnalyzeArgs), + Vacuum(VacuumArgs), +} + +impl DbCommands { + pub async fn run(&self) -> UtilesResult<()> { + match self { + DbCommands::Analyze(args) => analyze_main(args).await, + DbCommands::Vacuum(args) => vacuum_main(args).await, + } + } +} + #[derive(Debug, Parser)] pub struct AnalyzeArgs { #[command(flatten)] @@ -232,6 +279,7 @@ pub struct AnalyzeArgs { } #[derive(Debug, Parser)] +/// vacuum sqlite db inplace/into pub struct VacuumArgs { #[command(flatten)] pub common: SqliteDbCommonArgs, @@ -313,6 +361,21 @@ pub struct InfoArgs { pub(crate) full: bool, } +#[derive(Debug, Parser)] +pub struct AggHashArgs { + #[command(flatten)] + pub common: SqliteDbCommonArgs, + + #[command(flatten)] + pub filter_args: TilesFilterArgs, + // /// bbox(es) (west, south, east, north) + // #[arg(required = false, long, value_parser = parse_bbox_ext, allow_hyphen_values = true)] + // pub bbox: Option>, + /// hash to use for blob-id if copying to normal/hash db type + #[arg(required = false, long)] + pub hash: Option, +} + #[derive(Debug, Parser)] pub struct UpdateArgs { #[command(flatten)] @@ -340,6 +403,9 @@ pub enum Commands { #[command(name = "about", visible_alias = "aboot")] About, + #[command(subcommand)] + Db(DbCommands), + /// Echo the `tile.json` for mbtiles file #[command(name = "tilejson", visible_alias = "tj", alias = "trader-joes")] Tilejson(TilejsonArgs), @@ -356,6 +422,10 @@ pub enum Commands { #[command(name = "lint")] Lint(LintArgs), + /// Agg hash db + #[command(name = "agg-hash")] + AggHash(AggHashArgs), + /// Echo metadata (table) as json arr/obj #[command(name = "metadata", visible_aliases = ["meta", "md"])] Metadata(MetadataArgs), @@ -376,7 +446,6 @@ pub enum Commands { #[command(name = "info")] Info(InfoArgs), - /// VACUUM sqlite db #[command(name = "vacuum", visible_alias = "vac")] Vacuum(VacuumArgs), @@ -621,11 +690,11 @@ pub struct ZoomArgGroup { pub zoom: Option>>, /// min zoom level (0-30) - #[arg(long, conflicts_with = "zoom")] + #[arg(long, conflicts_with = "zoom", aliases = ["min-zoom", "min-z"])] pub minzoom: Option, /// max zoom level (0-30) - #[arg(long, conflicts_with = "zoom")] + #[arg(long, conflicts_with = "zoom", aliases = ["max-zoom", "max-z"])] pub maxzoom: Option, } @@ -647,6 +716,7 @@ impl ZoomArgGroup { #[derive( Debug, Copy, Parser, Clone, clap::ValueEnum, strum::EnumString, AsRefStr, Default, )] +#[strum(serialize_all = "kebab-case")] pub enum ConflictStrategy { #[default] Undefined, @@ -698,10 +768,14 @@ pub struct CopyArgs { #[arg(required = false, long, short, default_value = "undefined")] pub conflict: ConflictStrategy, - /// db-type - #[arg(required = false, long)] + /// db-type (default: src type) + #[arg(required = false, long = "dbtype", aliases = ["db-type", "mbtype", "mbt-type"])] pub dbtype: Option, + /// hash to use for blob-id if copying to normal/hash db type + #[arg(required = false, long)] + pub hash: Option, + /// n-jobs ~ 0=ncpus (default: max(4, ncpus)) #[arg(required = false, long, short)] pub jobs: Option, @@ -736,23 +810,3 @@ impl CopyArgs { } } } - -impl From<&CopyArgs> for CopyConfig { - fn from(args: &CopyArgs) -> CopyConfig { - let dbtype = args.dbtype.as_ref().map(|dbtype| dbtype.into()); - CopyConfig { - src: PathBuf::from(&args.src), - dst: PathBuf::from(&args.dst), - zset: args.zoom_set(), - zooms: args.zooms(), - verbose: true, - bboxes: args.bboxes(), - bounds_string: args.bounds(), - force: false, - dryrun: false, - jobs: args.jobs, - istrat: args.conflict.into(), - dbtype, - } - } -} diff --git a/crates/utiles/src/cli/commands/about.rs b/crates/utiles/src/cli/commands/about.rs index a478d5fc..49b1c360 100644 --- a/crates/utiles/src/cli/commands/about.rs +++ b/crates/utiles/src/cli/commands/about.rs @@ -1,20 +1,22 @@ -use crate::UtilesResult; +use crate::errors::UtilesResult; pub fn about_main() -> UtilesResult<()> { let current_exe = std::env::current_exe().unwrap_or_else(|_| std::path::PathBuf::from("unknown")); - println!("{} ~ {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); - println!("authors: {}", env!("CARGO_PKG_AUTHORS")); - println!("desc: {}", env!("CARGO_PKG_DESCRIPTION")); - println!("repo: {}", env!("CARGO_PKG_REPOSITORY")); - println!("which: {}", current_exe.display()); - println!( - "profile: {}", - if cfg!(debug_assertions) { - "debug" - } else { - "release" - } - ); + let prof = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + let parts = [ + format!("{} ~ {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")), + format!("version: {}", env!("CARGO_PKG_VERSION")), + format!("authors: {}", env!("CARGO_PKG_AUTHORS")), + format!("desc: {}", env!("CARGO_PKG_DESCRIPTION")), + format!("repo: {}", env!("CARGO_PKG_REPOSITORY")), + format!("which: {}", current_exe.display()), + format!("profile: {prof}"), + ]; + println!("{}", parts.join("\n")); Ok(()) } diff --git a/crates/utiles/src/cli/commands/agg_hash.rs b/crates/utiles/src/cli/commands/agg_hash.rs new file mode 100644 index 00000000..9115bae0 --- /dev/null +++ b/crates/utiles/src/cli/commands/agg_hash.rs @@ -0,0 +1,18 @@ +use crate::cli::args::AggHashArgs; +use crate::errors::UtilesResult; +use crate::mbt::hash_types::HashType; +use crate::mbt::mbt_agg_tiles_hash; +use crate::sqlite::AsyncSqliteConn; +use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; + +pub async fn agg_hash_main(args: &AggHashArgs) -> UtilesResult<()> { + let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?; + mbt.register_utiles_sqlite_functions().await?; + let hash_type = args.hash.unwrap_or(HashType::Md5); + let filter = args.filter_args.tiles_filter_maybe(); + let result = mbt + .conn(move |c| Ok(mbt_agg_tiles_hash(c, hash_type, None, &filter))) + .await??; + println!("{}", serde_json::to_string_pretty(&result)?); + Ok(()) +} diff --git a/crates/utiles/src/cli/commands/children_parent.rs b/crates/utiles/src/cli/commands/children_parent.rs index 3001cc65..db9bdeb8 100644 --- a/crates/utiles/src/cli/commands/children_parent.rs +++ b/crates/utiles/src/cli/commands/children_parent.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unwrap_used)] use utiles_core::{Tile, TileLike}; use crate::cli::args::ParentChildrenArgs; diff --git a/crates/utiles/src/cli/commands/dev.rs b/crates/utiles/src/cli/commands/dev.rs index e4eed65b..7030084c 100644 --- a/crates/utiles/src/cli/commands/dev.rs +++ b/crates/utiles/src/cli/commands/dev.rs @@ -1,5 +1,5 @@ use clap::Parser; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; use crate::errors::UtilesResult; use crate::mbt::hash_types::HashType; @@ -20,11 +20,7 @@ pub struct DevArgs { fspath: Option, } -#[allow(clippy::unused_async)] -async fn dev(args: DevArgs) -> UtilesResult<()> { - // DEV START - debug!("args: {:?}", args); - let filepath = args.fspath.unwrap(); +fn _timing_agg_tiles_hash(filepath: &str) -> UtilesResult<()> { let mbt = Mbtiles::open(filepath)?; add_functions(&mbt.conn)?; let hashes = vec![ @@ -37,14 +33,29 @@ async fn dev(args: DevArgs) -> UtilesResult<()> { ]; for hash in hashes { let start_time = std::time::Instant::now(); - let agg_tile_hash = mbt_agg_tiles_hash(&mbt.conn, hash)?; + let agg_tile_hash = mbt_agg_tiles_hash(&mbt.conn, hash, None, &None)?; let elapsed = start_time.elapsed(); debug!("---------------------"); debug!("hash: {:?}, agg_tile_hash: {:?}", hash, agg_tile_hash); debug!("agg_tile_hash: {:?}", agg_tile_hash); debug!("elapsed: {:?}", elapsed); } + Ok(()) +} +#[allow(clippy::unused_async)] +async fn dev(args: DevArgs) -> UtilesResult<()> { + // DEV START + debug!("args: {:?}", args); + match args.fspath { + Some(filepath) => { + info!("fspath: {:?}", filepath); + // timing_agg_tiles_hash(&filepath)?; + } + None => { + warn!("no fspath provided"); + } + } // DEV END Ok(()) } diff --git a/crates/utiles/src/cli/commands/info.rs b/crates/utiles/src/cli/commands/info.rs index e750b843..89518909 100644 --- a/crates/utiles/src/cli/commands/info.rs +++ b/crates/utiles/src/cli/commands/info.rs @@ -1,29 +1,30 @@ +use std::fs::canonicalize; use std::path::Path; use crate::cli::args::InfoArgs; use crate::errors::UtilesResult; +use crate::fs_async::file_exists; use crate::mbt::MbtilesStats; -use crate::utilesqlite::Mbtiles; +use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; -fn mbinfo(filepath: &str) -> UtilesResult { - let filepath = Path::new(filepath); - assert!( - filepath.exists(), - "File does not exist: {}", - filepath.display() - ); - assert!( - filepath.is_file(), - "Not a file: {filepath}", - filepath = filepath.display() - ); - let mbtiles: Mbtiles = Mbtiles::from(filepath); - let stats = mbtiles.mbt_stats(None)?; - Ok(stats) +async fn mbinfo(filepath: &str) -> UtilesResult { + let fspath = Path::new(filepath); + let is_file = file_exists(filepath).await; + if is_file { + let mbt = MbtilesAsyncSqliteClient::open_existing(filepath).await?; + let stats = mbt.mbt_stats(None).await?; + Ok(stats) + } else { + let abspath = canonicalize(fspath)?; + + Err(crate::errors::UtilesError::NotAFile( + abspath.to_string_lossy().to_string(), + )) + } } -pub fn info_main(args: &InfoArgs) -> UtilesResult<()> { - let stats = mbinfo(&args.common.filepath)?; +pub async fn info_main(args: &InfoArgs) -> UtilesResult<()> { + let stats = mbinfo(&args.common.filepath).await?; let str = if args.common.min { serde_json::to_string(&stats) } else { diff --git a/crates/utiles/src/cli/commands/lint.rs b/crates/utiles/src/cli/commands/lint.rs index c3cf91e1..f8a5e205 100644 --- a/crates/utiles/src/cli/commands/lint.rs +++ b/crates/utiles/src/cli/commands/lint.rs @@ -1,208 +1,9 @@ -use std::path::PathBuf; - -use futures::{stream, StreamExt}; use tracing::{debug, warn}; use crate::cli::args::LintArgs; use crate::errors::UtilesResult; use crate::globster; -use crate::lint::MbtilesLinter; - -// pub fn lint_mbtiles_file(mbtiles: &Mbtiles, _fix: bool) -> Vec { -// let mut errors = Vec::new(); -// // match mbtiles.magic_number() { -// // Ok(magic_number) => { -// // match magic_number { -// // MBTILES_MAGIC_NUMBER => {} -// // // zero -// // 0 => { -// // errors.push(UtilesLintError::MbtMissingMagicNumber); -// // } -// // _ => { -// // errors.push(UtilesLintError::MbtUnknownMagicNumber(magic_number)); -// // } -// // } -// // } -// // Err(e) => { -// // errors.push(UtilesLintError::Unknown(e.to_string())); -// // } -// // } -// -// // let mbtiles = mbtiles_result.unwrap(); -// let has_unique_index_on_metadata_name = -// mbtiles.has_unique_index_on_metadata().unwrap(); -// let metadata_name_is_primary_key = -// mbtiles.metadata_table_name_is_primary_key().unwrap(); -// -// let rows = mbtiles.metadata().unwrap(); -// -// if has_unique_index_on_metadata_name || metadata_name_is_primary_key { -// let duplicate_rows = metadata2duplicates(rows.clone()); -// if !duplicate_rows.is_empty() { -// errors.extend( -// duplicate_rows -// .keys() -// .map(|k| UtilesLintError::DuplicateMetadataKey(k.clone())) -// .collect::>(), -// ); -// } -// } else { -// errors.push(UtilesLintError::MissingUniqueIndex( -// "metadata.name".to_string(), -// )); -// } -// let map = metadata2map(&rows); -// let map_errs = lint_metadata_map(&map); -// if !map_errs.is_empty() { -// errors.extend(map_errs); -// } -// errors -// } - -// pub fn lint_filepath( -// fspath: &Path, -// fix: bool, -// ) -> UtilesLintResult> { -// let Some(fspath_str) = fspath.to_str() else { -// return Err(UtilesLintError::InvalidPath( -// fspath.to_str().unwrap().to_string(), -// )); -// }; -// // let fspath_str = match fspath.to_str() { -// // Some(s) => s, -// // None => { -// // return Err(UtilesLintError::InvalidPath( -// // fspath.to_str().unwrap().to_string(), -// // )) -// // } -// // }; -// -// if !fspath_str.ends_with(".mbtiles") { -// let conn = match squealite::open(fspath_str) { -// Ok(conn) => conn, -// Err(e) => { -// warn!("Unable to open file: {}", e); -// return Err(UtilesLintError::UnableToOpen(fspath_str.to_string())); -// } -// }; -// -// match is_mbtiles(&conn) { -// Ok(false) => return Ok(vec![]), -// Ok(true) => { -// let mbtiles = Mbtiles::from_conn(conn); -// return Ok(lint_mbtiles_file(&mbtiles, fix)); -// } -// Err(e) => { -// warn!("Unable to determine if file is mbtiles: {}", e); -// return Err(UtilesLintError::NotAMbtilesDb(fspath_str.to_string())); -// } -// } -// } -// -// match Mbtiles::from_filepath_str(fspath_str) { -// Ok(mbtiles) => Ok(lint_mbtiles_file(&mbtiles, fix)), -// Err(e) => { -// warn!("ERROR: {}", e); -// Err(UtilesLintError::UnableToOpen(fspath_str.to_string())) -// } -// } -// } - -// async fn lint_filepath_async( -// path: &Path, -// fix: bool, -// ) -> UtilesLintResult> { -// let linter = MbtilesLinter::new(path, fix); -// -// linter.lint().await -// } - -async fn lint_filepaths(fspaths: Vec, fix: bool) { - // for each concurrent - - stream::iter(fspaths) - .for_each_concurrent(4, |path| async move { - let linter = MbtilesLinter::new(&path, fix); - let lint_results = linter.lint().await; - match lint_results { - Ok(r) => { - debug!("r: {:?}", r); - // print each err.... - if r.is_empty() { - debug!("OK: {}", path.display()); - } else { - warn!("{} - {} errors found", path.display(), r.len()); - // let agg_err = UtilesLintError::LintErrors(r); - let strings = r - .iter() - .map(|e| e.format_error(&path.display().to_string())) - .collect::>(); - let joined = strings.join("\n"); - println!("{joined}"); - for err in r { - warn!("{}", err.to_string()); - } - } - } - Err(e) => { - warn!("Error: {}", e); - } - } - }) - .await; - - // for path in fspaths { - // let r = lint_filepath(&path, fix); - // match r { - // Ok(r) => { - // debug!("r: {:?}", r); - // // print each err.... - // if r.is_empty() { - // info!("No errors found"); - // } else { - // warn!("{} - {} errors found", path.display(), r.len()); - // - // // let agg_err = UtilesLintError::LintErrors(r); - // for err in r { - // warn!("{}", err.to_string()); - // } - // } - // } - // Err(e) => { - // warn!("Unable to open file: {}", e); - // warn!("Error: {}", e); - // } - // } - // } - - // ============ - // SYNC VERSION - // ============ - // for path in fspaths { - // let linter = mbtileslinter::new(&path, fix); - // let lint_results = linter.lint().await; - // match lint_results { - // ok(r) => { - // debug!("r: {:?}", r); - // // print each err.... - // if r.is_empty() { - // debug!("{} - ok", path.display()); - // } else { - // warn!("{} - {} errors found", path.display(), r.len()); - // - // // let agg_err = utileslinterror::linterrors(r); - // for err in r { - // warn!("{}", err.to_string()); - // } - // } - // } - // err(e) => { - // warn!("unable to open file: {}", e); - // warn!("error: {}", e); - // } - // } - // } -} +use crate::lint::lint_filepaths; pub async fn lint_main(args: &LintArgs) -> UtilesResult<()> { let filepaths = globster::find_filepaths(&args.fspaths)?; @@ -214,6 +15,6 @@ pub async fn lint_main(args: &LintArgs) -> UtilesResult<()> { warn!("No files found"); return Ok(()); } - lint_filepaths(filepaths, args.fix).await; + lint_filepaths(filepaths, args.fix).await?; Ok(()) } diff --git a/crates/utiles/src/cli/commands/metadata.rs b/crates/utiles/src/cli/commands/metadata.rs index 7e206bfd..cf848237 100644 --- a/crates/utiles/src/cli/commands/metadata.rs +++ b/crates/utiles/src/cli/commands/metadata.rs @@ -3,12 +3,13 @@ use std::path::Path; use crate::cli::args::{MetadataArgs, MetadataSetArgs}; use crate::cli::stdin2string::stdin2string; use crate::errors::UtilesResult; +use crate::fs_async::file_exists_err; use crate::mbt::{ metadata2map, metadata2map_val, read_metadata_json, MbtilesMetadataJson, MbtilesMetadataRowParsed, MetadataChange, }; -use crate::utilesqlite::{Mbtiles, MbtilesAsync, MbtilesAsyncSqliteClient}; -use serde::Serialize; +use crate::sqlite::AsyncSqliteConn; +use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use tracing::warn; use tracing::{debug, info}; @@ -18,7 +19,6 @@ pub async fn metadata_main(args: &MetadataArgs) -> UtilesResult<()> { let filepath = Path::new(&args.common.filepath); let mbtiles = MbtilesAsyncSqliteClient::open_existing(filepath).await?; let metadata_rows = mbtiles.metadata_rows().await?; - let json_val = match (args.raw, args.obj) { (true, true) => { let m = metadata2map(&metadata_rows); @@ -38,44 +38,27 @@ pub async fn metadata_main(args: &MetadataArgs) -> UtilesResult<()> { } }; let out_str = if args.common.min { - serde_json::to_string::(&json_val).unwrap() + serde_json::to_string::(&json_val) } else { - serde_json::to_string_pretty::(&json_val).unwrap() - }; + serde_json::to_string_pretty::(&json_val) + }?; + println!("{out_str}"); Ok(()) } -#[derive(Debug, Serialize)] -pub struct MetadataChangeFromTo { - pub name: String, - pub from: Option, - pub to: Option, -} - pub async fn metadata_set_main(args: &MetadataSetArgs) -> UtilesResult<()> { debug!("meta: {}", args.common.filepath); // check that filepath exists and is file let filepath = Path::new(&args.common.filepath); - assert!( - filepath.exists(), - "File does not exist: {}", - filepath.display() - ); - assert!( - filepath.is_file(), - "Not a file: {filepath}", - filepath = filepath.display() - ); - - let mbtiles: Mbtiles = Mbtiles::from(filepath); - let current_metadata_json = mbtiles.metadata_json()?; + file_exists_err(filepath).await?; + let mbtiles = MbtilesAsyncSqliteClient::open_existing(filepath).await?; + let current_metadata_json = mbtiles.metadata_json().await?; let c = match &args.value { Some(value) => { let mut mdjson = current_metadata_json.clone(); mdjson.insert(&args.key, value); - let (forward, inverse, data) = current_metadata_json.diff(&mdjson, true)?; - MetadataChange::from_forward_reverse_data(forward, inverse, data) + current_metadata_json.diff(&mdjson, true)? } None => { // check if key is filepath ending in .json then load and @@ -86,26 +69,17 @@ pub async fn metadata_set_main(args: &MetadataSetArgs) -> UtilesResult<()> { let mdjson = read_metadata_json(&args.key).await?; debug!("mdjson: {:?}", mdjson); - let (forward, inverse, data) = - current_metadata_json.diff(&mdjson, true)?; - MetadataChange::from_forward_reverse_data(forward, inverse, data) + current_metadata_json.diff(&mdjson, true)? } else if args.key.to_lowercase() == "-" || args.key.to_lowercase() == "--" { // get metadata from stdin... let stdin_str = stdin2string()?; let mdjson = serde_json::from_str::(&stdin_str)?; - let (forward, inverse, data) = - current_metadata_json.diff(&mdjson, true)?; - - MetadataChange::from_forward_reverse_data(forward, inverse, data) + current_metadata_json.diff(&mdjson, true)? } else { let mut mdjson = current_metadata_json.clone(); mdjson.delete(&args.key); - - let (forward, inverse, data) = - current_metadata_json.diff(&mdjson, false)?; - - MetadataChange::from_forward_reverse_data(forward, inverse, data) + current_metadata_json.diff(&mdjson, true)? } } }; @@ -120,7 +94,11 @@ pub async fn metadata_set_main(args: &MetadataSetArgs) -> UtilesResult<()> { if args.dryrun { warn!("Dryrun: no changes made"); } else { - MetadataChange::apply_changes_to_connection(&mbtiles.conn, &vec![c])?; + mbtiles + .conn(|conn| { + MetadataChange::apply_changes_to_connection(conn, &vec![c]) + }) + .await?; } } Ok(()) diff --git a/crates/utiles/src/cli/commands/mod.rs b/crates/utiles/src/cli/commands/mod.rs index 845ed5a6..e4e8e1b2 100644 --- a/crates/utiles/src/cli/commands/mod.rs +++ b/crates/utiles/src/cli/commands/mod.rs @@ -1,8 +1,10 @@ -#![allow(clippy::unwrap_used)] +// #![allow(clippy::unwrap_used)] pub use about::about_main; +pub use agg_hash::agg_hash_main; pub use children_parent::{children_main, parent_main}; pub use contains::contains_main; pub use copy::copy_main; +pub use db::analyze_main; pub use dev::dev_main; pub use info::info_main; pub use lint::lint_main; @@ -21,6 +23,7 @@ pub use vacuum::vacuum_main; pub use zxyify::zxyify_main; mod about; +mod agg_hash; mod children_parent; mod contains; pub mod copy; diff --git a/crates/utiles/src/cli/commands/rimraf.rs b/crates/utiles/src/cli/commands/rimraf.rs index d3498baa..d7a786be 100644 --- a/crates/utiles/src/cli/commands/rimraf.rs +++ b/crates/utiles/src/cli/commands/rimraf.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unwrap_used)] use std::cell::Cell; use std::path::Path; diff --git a/crates/utiles/src/cli/commands/serve.rs b/crates/utiles/src/cli/commands/serve.rs index 3cd66aae..057e7266 100644 --- a/crates/utiles/src/cli/commands/serve.rs +++ b/crates/utiles/src/cli/commands/serve.rs @@ -1,7 +1,7 @@ -use crate::errors::UtilesResult; use clap::Parser; use tracing::{debug, warn}; +use crate::errors::UtilesResult; use crate::server::{utiles_serve, UtilesServerConfig}; #[derive(Debug, Parser)] @@ -10,9 +10,6 @@ pub struct ServeArgs { #[arg(required = false)] fspaths: Option>, - // /// config fspath (TODO) - // #[arg(long, short = 'c')] - // config: Option, /// Port to server on #[arg(long, short = 'p', default_value = "3333")] port: u16, @@ -39,10 +36,19 @@ impl ServeArgs { #[allow(clippy::unused_async)] pub async fn serve_main(args: ServeArgs) -> UtilesResult<()> { debug!("args: {:?}", args); - if args.fspaths.is_none() || args.fspaths.as_ref().unwrap().is_empty() { + if let Some(ref fspaths) = args.fspaths { + if fspaths.is_empty() { + warn!("fspaths is empty"); + } + for fspath in fspaths { + if !std::path::Path::new(fspath).exists() { + warn!("fspath does not exist: {:?}", fspath); + } + } + } else { warn!("no fspaths provided"); } let cfg = args.to_cfg(); - utiles_serve(cfg).await.expect("utiles_serve failed"); + utiles_serve(cfg).await?; Ok(()) } diff --git a/crates/utiles/src/cli/commands/shapes.rs b/crates/utiles/src/cli/commands/shapes.rs index 3b323f59..6f8490e5 100644 --- a/crates/utiles/src/cli/commands/shapes.rs +++ b/crates/utiles/src/cli/commands/shapes.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unwrap_used)] use clap::{Args, Parser}; use serde_json::{Map, Value}; use tracing::{debug, error}; @@ -7,7 +8,8 @@ use utiles_core::tile::FeatureOptions; use utiles_core::Tile; use crate::cli::stdinterator::StdInterator; -use crate::errors::{UtilesError, UtilesResult}; +use crate::errors::UtilesError; +use crate::errors::UtilesResult; // #[group(required = false, id="projected")] #[derive(Args, Debug)] @@ -152,28 +154,6 @@ pub fn shapes_main(args: ShapesArgs) -> UtilesResult<()> { Err(UtilesError::Error(format!("Error parsing tile: {e}"))) } } - // Some( - // TileWithProperties { - // tile: t, - // id, - // properties, - // } - // ) - // match t { - // Ok(tile) => { - // let tile_with_properties = TileWithProperties { - // tile, - // id, - // properties, - // }; - // Some(tile_with_properties) - // } - // Err(e) => { - // error!("Error parsing tile: {}", e); - // // throw the error here - // panic!("Error parsing tile: {}", e); - // } - // } }); let feature_options: FeatureOptions = FeatureOptions { fid: None, diff --git a/crates/utiles/src/cli/commands/tile_stream_cmds.rs b/crates/utiles/src/cli/commands/tile_stream_cmds.rs index b6395f26..262d9067 100644 --- a/crates/utiles/src/cli/commands/tile_stream_cmds.rs +++ b/crates/utiles/src/cli/commands/tile_stream_cmds.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unwrap_used)] use tracing::error; use utiles_core::{bounding_tile, Tile, TileLike}; @@ -82,7 +83,9 @@ pub fn quadkey_main(args: TileFmtArgs) -> UtilesResult<()> { // if the line begins w/ '['/'{' treat as json-tile // otherwise treat as quadkey let lstr = line.unwrap(); - let first_char = lstr.chars().next().unwrap(); + let maybe_first_char = lstr.chars().next(); + let first_char = maybe_first_char.unwrap(); + // .unwrap(); match first_char { '[' | '{' => { // treat as tile diff --git a/crates/utiles/src/cli/commands/tilejson.rs b/crates/utiles/src/cli/commands/tilejson.rs index 0c7d7d87..a07f08f3 100644 --- a/crates/utiles/src/cli/commands/tilejson.rs +++ b/crates/utiles/src/cli/commands/tilejson.rs @@ -1,34 +1,18 @@ -use std::path::Path; - use tracing::debug; -use crate::utilejson::tilejson_stringify; -use crate::utilesqlite::Mbtiles; - use crate::cli::args::TilejsonArgs; use crate::errors::UtilesResult; +use crate::utilejson::tilejson_stringify; +use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; -pub fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> { +pub async fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> { debug!("tilejson: {}", args.common.filepath); - // check that filepath exists and is file - let filepath = Path::new(&args.common.filepath); - assert!( - filepath.exists(), - "File does not exist: {}", - filepath.display() - ); - assert!( - filepath.is_file(), - "Not a file: {filepath}", - filepath = filepath.display() - ); - let mbtiles: Mbtiles = Mbtiles::from(filepath); - let mut tj = mbtiles.tilejson().unwrap(); + let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?; + let mut tj = mbt.tilejson().await?; if !args.tilestats { tj.other.remove("tilestats"); } let s = tilejson_stringify(&tj, Option::from(!args.common.min)); println!("{s}"); - Ok(()) } diff --git a/crates/utiles/src/cli/commands/tiles.rs b/crates/utiles/src/cli/commands/tiles.rs index 281afafa..f26ae599 100644 --- a/crates/utiles/src/cli/commands/tiles.rs +++ b/crates/utiles/src/cli/commands/tiles.rs @@ -1,5 +1,5 @@ -use std::io; -use std::io::Write; +use std::io::BufWriter; +use std::{io, io::Write}; use tracing::{debug, error}; @@ -20,9 +20,8 @@ pub async fn tiles_main( let lines = stdin_filtered(args.inargs.input); let mut stdout = io::stdout(); let lock = stdout.lock(); - let mut buf = std::io::BufWriter::with_capacity(32 * 1024, lock); - - let tiles = lines + let mut buf = BufWriter::with_capacity(32 * 1024, lock); + let tiles_iterators = lines .map(|l| match l { Ok(s) => { debug!("l: {:?}", s); @@ -43,27 +42,29 @@ pub async fn tiles_main( }) }) }) - .map(|result| result.unwrap()) - .flat_map(|b| { - tiles( - (b.west, b.south, b.east, b.north), - ZoomOrZooms::Zoom(args.zoom), - ) - }) - .enumerate(); + .map(|parse_result| { + parse_result.map(|b| { + tiles( + (b.west, b.south, b.east, b.north), + ZoomOrZooms::Zoom(args.zoom), + ) + }) + }); let formatter = TileStringFormatter::from(&args.fmtopts); let rs = if args.fmtopts.seq { "\x1e\n" } else { "" }; - for (i, tile) in tiles { - let tile_str = formatter.fmt_tile(&tile); - let out_str = format!("{rs}{tile_str}\n"); - buf.write_all(out_str.as_bytes())?; - // writeln!(stdout, "{}{}", rs, tile_fmt.format_tile(&tile)).unwrap(); - // call loop_fn if it's defined every 1000 iterations for signal break - if i % 1024 == 0 { - stdout.flush()?; - tokio::time::sleep(tokio::time::Duration::from_secs(0)).await; - if let Some(f) = loop_fn { - f(); + for tiles_iterator in tiles_iterators { + let tiles_iterator = tiles_iterator?; + for (i, tile) in tiles_iterator.enumerate() { + let tile_str = formatter.fmt_tile(&tile); + let out_str = format!("{rs}{tile_str}\n"); + buf.write_all(out_str.as_bytes())?; + // call loop_fn if it's defined every 1000 iterations for signal break + if i % 1024 == 0 { + stdout.flush()?; + tokio::time::sleep(tokio::time::Duration::from_secs(0)).await; + if let Some(f) = loop_fn { + f(); + } } } } diff --git a/crates/utiles/src/cli/commands/touch.rs b/crates/utiles/src/cli/commands/touch.rs index e46b1b3c..7cce620e 100644 --- a/crates/utiles/src/cli/commands/touch.rs +++ b/crates/utiles/src/cli/commands/touch.rs @@ -1,14 +1,15 @@ use std::path::Path; +use chrono; +use tracing::{debug, info}; + use crate::cli::args::TouchArgs; use crate::errors::UtilesResult; use crate::mbt::MbtType; -use crate::sqlite::SqliteError; -use crate::sqlite::{is_valid_page_size, Sqlike3}; -use crate::utilesqlite::Mbtiles; +use crate::sqlite::is_valid_page_size; +use crate::sqlite::{Sqlike3Async, SqliteError}; +use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use crate::UtilesError; -use chrono; -use tracing::{debug, info}; fn check_page_size(page_size: i64) -> UtilesResult { if is_valid_page_size(page_size) { @@ -19,9 +20,8 @@ fn check_page_size(page_size: i64) -> UtilesResult { } } -pub fn touch_main(args: &TouchArgs) -> UtilesResult<()> { +pub async fn touch_main(args: &TouchArgs) -> UtilesResult<()> { let filepath = &args.filepath; - let page_size = check_page_size(args.page_size.unwrap_or(4096))?; debug!("touch: {}", filepath); let dbtype: MbtType = args.mbtype(); @@ -32,20 +32,40 @@ pub fn touch_main(args: &TouchArgs) -> UtilesResult<()> { if fpth.extension().is_none() { Err(UtilesError::NoFspathExtension(filepath.to_string())) } else { - let filename_no_ext = fpth.file_stem().unwrap().to_str().unwrap(); - - assert!(!fpth.exists(), "Already exists: {}", fpth.display()); - let mbtiles = Mbtiles::create(filepath, Some(dbtype))?; - info!("Created mbtiles: {:?}", filepath); - mbtiles.metadata_set("name", filename_no_ext)?; - mbtiles.metadata_set("mbtype", dbtype_str.to_string().as_str())?; - // current iso datetimestamp - let now = chrono::Utc::now(); - let now_str = now.to_rfc3339(); - mbtiles.metadata_set("ctime", now_str.as_str())?; - mbtiles.metadata_set("mtime", now_str.as_str())?; - mbtiles.pragma_page_size_set(page_size)?; - mbtiles.vacuum()?; - Ok(()) + let stem_str = fpth.file_stem(); + if stem_str.is_none() { + Err(UtilesError::NoFspathExtension(filepath.to_string())) + } else { + let filename_no_ext = stem_str; + match filename_no_ext { + Some(filename_no_ext) => { + let filename_no_ext = { + match filename_no_ext.to_str() { + Some(filename_no_ext) => filename_no_ext.to_string(), + None => { + return Err(UtilesError::PathConversionError( + filepath.to_string(), + )); + } + } + }; + let mbtiles = + MbtilesAsyncSqliteClient::open_new(filepath, Some(dbtype)) + .await?; + info!("Created mbtiles: {:?}", filepath); + mbtiles.metadata_set("name", &filename_no_ext).await?; + mbtiles.metadata_set("mbtype", dbtype_str.as_str()).await?; + // current iso datetimestamp + let now = chrono::Utc::now(); + let now_str = now.to_rfc3339(); + mbtiles.metadata_set("ctime", now_str.as_str()).await?; + mbtiles.metadata_set("mtime", now_str.as_str()).await?; + mbtiles.pragma_page_size_set(page_size).await?; + mbtiles.vacuum().await?; + Ok(()) + } + None => Err(UtilesError::NoFspathStem(filepath.to_string())), + } + } } } diff --git a/crates/utiles/src/cli/commands/update.rs b/crates/utiles/src/cli/commands/update.rs index f055ff5e..60783064 100644 --- a/crates/utiles/src/cli/commands/update.rs +++ b/crates/utiles/src/cli/commands/update.rs @@ -3,18 +3,19 @@ use std::path::Path; use tracing::{debug, warn}; use crate::cli::args::UpdateArgs; -use crate::cli::commands::metadata::MetadataChangeFromTo; use crate::errors::UtilesResult; +use crate::mbt::{DbChangeType, DbChangeset, MetadataChange, MetadataChangeFromTo}; use crate::sqlite::AsyncSqliteConn; -use crate::utilesqlite::mbtiles::query_distinct_tiletype_fast; -// use crate::utilesqlite::mbtiles_async_sqlite::AsyncSqlite; +use crate::utilesqlite::mbtiles::{ + query_distinct_tilesize_fast, query_distinct_tiletype_fast, +}; use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use crate::UtilesError; pub async fn update_mbtiles( filepath: &str, dryrun: bool, -) -> UtilesResult> { +) -> UtilesResult { // check that filepath exists and is file let mbt = if dryrun { MbtilesAsyncSqliteClient::open_readonly(filepath).await? @@ -28,15 +29,15 @@ pub async fn update_mbtiles( if tiles_is_empty { warn!("tiles table/view is empty: {}", filepath); } - let mut changes = vec![]; + let current_metadata = mbt.metadata_json().await?; + // ========================================================= // MINZOOM ~ MAXZOOM ~ MINZOOM ~ MAXZOOM ~ MINZOOM ~ MAXZOOM // ========================================================= let minzoom_maxzoom = mbt.query_minzoom_maxzoom().await?; - debug!("minzoom_maxzoom: {:?}", minzoom_maxzoom); - + // updated_metadata // Updating metadata... let metdata_minzoom = mbt.metadata_minzoom().await?; if let Some(minzoom_maxzoom) = minzoom_maxzoom { @@ -47,10 +48,6 @@ pub async fn update_mbtiles( from: Some(metadata_minzoom.to_string()), to: Some(minzoom_maxzoom.minzoom.to_string()), }); - if !dryrun { - mbt.metadata_set("minzoom", &minzoom_maxzoom.minzoom.to_string()) - .await?; - } } } else { changes.push(MetadataChangeFromTo { @@ -58,10 +55,6 @@ pub async fn update_mbtiles( from: None, to: Some(minzoom_maxzoom.minzoom.to_string()), }); - if !dryrun { - mbt.metadata_set("minzoom", &minzoom_maxzoom.minzoom.to_string()) - .await?; - } } } @@ -74,10 +67,6 @@ pub async fn update_mbtiles( from: Some(metadata_maxzoom.to_string()), to: Some(minzoom_maxzoom.maxzoom.to_string()), }); - if !dryrun { - mbt.metadata_set("maxzoom", &minzoom_maxzoom.maxzoom.to_string()) - .await?; - } } } else { changes.push(MetadataChangeFromTo { @@ -85,10 +74,6 @@ pub async fn update_mbtiles( from: None, to: Some(minzoom_maxzoom.maxzoom.to_string()), }); - if !dryrun { - mbt.metadata_set("maxzoom", &minzoom_maxzoom.maxzoom.to_string()) - .await?; - } } } @@ -101,18 +86,18 @@ pub async fn update_mbtiles( // register the fn mbt.register_utiles_sqlite_functions().await?; let format = mbt.metadata_row("format").await?; - let queryfmt = mbt + let query_fmt = mbt .conn( // whatever clone it! move |c| query_distinct_tiletype_fast(c, minmax), ) .await?; - match queryfmt.len() { + match query_fmt.len() { 0 => { warn!("no format found: {}", filepath); } 1 => { - let fmt = queryfmt[0].clone(); + let fmt = query_fmt[0].clone(); if let Some(format) = format { if format.value != fmt { changes.push(MetadataChangeFromTo { @@ -120,9 +105,6 @@ pub async fn update_mbtiles( from: Some(format.value.clone()), to: Some(fmt.clone()), }); - if !dryrun { - mbt.metadata_set("format", &fmt).await?; - } } } else { changes.push(MetadataChangeFromTo { @@ -130,19 +112,74 @@ pub async fn update_mbtiles( from: None, to: Some(fmt.clone()), }); - if !dryrun { - mbt.metadata_set("format", &fmt).await?; + } + } + _ => { + warn!("NOT IMPLEMENTED multiple formats found: {:?}", query_fmt); + } + } + + let tilesize = mbt.metadata_row("tilesize").await?; + let query_tilesize = mbt + .conn( + // whatever clone it! + move |c| query_distinct_tilesize_fast(c, minmax), + ) + .await?; + match query_tilesize.len() { + 0 => { + warn!("no tilesize found: {}", filepath); + } + 1 => { + let ts = query_tilesize[0]; + let ts_str: String = ts.to_string(); + if let Some(tilesize) = tilesize { + if tilesize.value != ts_str { + changes.push(MetadataChangeFromTo { + name: "tilesize".to_string(), + from: Some(tilesize.value.clone()), + to: Some(ts_str), + }); } + } else { + changes.push(MetadataChangeFromTo { + name: "tilesize".to_string(), + from: None, + to: Some(ts_str), + }); } } _ => { - warn!("NOT IMPLEMENTED multiple formats found: {:?}", queryfmt); + warn!( + "NOT IMPLEMENTED multiple tilesize found: {:?}", + query_tilesize + ); + } + } + + let metadata_change = if changes.is_empty() { + MetadataChange::new_empty() + } else { + let mut updated_metadata = current_metadata.clone(); + for change in &changes { + if let Some(new_val) = &change.to { + updated_metadata.insert(&change.name, new_val); + } } + + current_metadata.diff(&updated_metadata, true)? + }; + if dryrun { + warn!("Dryrun: no changes made"); + } else { + // todo fix cloning??? + let changes2apply = vec![metadata_change.clone()]; + mbt.conn(move |conn| { + MetadataChange::apply_changes_to_connection(conn, &changes2apply) + }) + .await?; } - debug!("queryfmt: {:?}", queryfmt); - debug!("metadata changes: {:?}", changes); - debug!("metdata_minzoom: {:?}", metdata_minzoom); - Ok(changes) + Ok(metadata_change) } pub async fn update_main(args: &UpdateArgs) -> UtilesResult<()> { @@ -159,9 +196,11 @@ pub async fn update_main(args: &UpdateArgs) -> UtilesResult<()> { filepath = filepath.display() ); let changes = update_mbtiles(&args.common.filepath, args.dryrun).await?; + debug!("changes: {:?}", changes); - let s = serde_json::to_string_pretty(&changes) - .expect("should not fail; changes is a Vec"); - println!("{s}"); + let db_changes = DbChangeset::from(DbChangeType::from(changes)); + let jsonstring = + serde_json::to_string_pretty(&db_changes).expect("should not fail"); + println!("{jsonstring}"); Ok(()) } diff --git a/crates/utiles/src/cli/commands/vacuum.rs b/crates/utiles/src/cli/commands/vacuum.rs index 1bbf4038..85240e12 100644 --- a/crates/utiles/src/cli/commands/vacuum.rs +++ b/crates/utiles/src/cli/commands/vacuum.rs @@ -3,7 +3,8 @@ use tracing::{debug, error, info, warn}; use crate::cli::args::VacuumArgs; use crate::errors::UtilesResult; -use crate::sqlite::{Sqlike3, SqliteDb}; +use crate::fs_async::filesize_async; +use crate::sqlite::{Sqlike3Async, SqliteDbAsyncClient}; use crate::UtilesError; #[derive(Debug, Serialize, Deserialize)] @@ -21,17 +22,17 @@ pub struct VacuumInfo { pub size_diff: i64, } -pub fn vacuum_main(args: &VacuumArgs) -> UtilesResult<()> { +pub async fn vacuum_main(args: &VacuumArgs) -> UtilesResult<()> { // check that the file exists - let db = SqliteDb::open_existing(&args.common.filepath)?; + let db = SqliteDbAsyncClient::open_existing(&args.common.filepath, None).await?; let pre_vac_file_size = std::fs::metadata(&args.common.filepath)?.len(); if args.page_size.is_some() { - let current_page_size = db.pragma_page_size()?; + let current_page_size = db.pragma_page_size().await?; if let Some(page_size) = args.page_size { if current_page_size != page_size { debug!("setting page size: {} -> {}", current_page_size, page_size); - db.pragma_page_size_set(page_size)?; + db.pragma_page_size_set(page_size).await?; } } } @@ -51,10 +52,10 @@ pub fn vacuum_main(args: &VacuumArgs) -> UtilesResult<()> { } } info!("vacuuming: {} -> {}", args.common.filepath, dst); - db.vacuum_into(dst.clone())?; + db.vacuum_into(dst.clone()).await?; } else { info!("vacuuming: {}", args.common.filepath); - db.vacuum()?; + db.vacuum().await?; } let vacuum_time_ms = vacuum_start_time.elapsed().as_millis(); @@ -63,20 +64,20 @@ pub fn vacuum_main(args: &VacuumArgs) -> UtilesResult<()> { if args.analyze { let analyze_start_time = std::time::Instant::now(); if let Some(dst) = &args.into { - let dst_db = SqliteDb::open_existing(dst)?; + let dst_db = SqliteDbAsyncClient::open_existing(dst, None).await?; info!("analyzing: {}", dst); - dst_db.analyze()?; + dst_db.analyze().await?; } else { info!("analyzing: {}", args.common.filepath); - db.analyze()?; + db.analyze().await?; } analyze_time_ms = analyze_start_time.elapsed().as_millis(); } // get file size from filepath let vacuumed_file_size = match &args.into { - Some(dst) => std::fs::metadata(dst)?.len(), - None => std::fs::metadata(&args.common.filepath)?.len(), + Some(dst) => filesize_async(dst).await.unwrap_or(0), + None => filesize_async(&args.common.filepath).await.unwrap_or(0), }; let info = VacuumInfo { fspath: args.common.filepath.clone(), diff --git a/crates/utiles/src/cli/commands/zxyify.rs b/crates/utiles/src/cli/commands/zxyify.rs index 2091866a..c5a9cdd6 100644 --- a/crates/utiles/src/cli/commands/zxyify.rs +++ b/crates/utiles/src/cli/commands/zxyify.rs @@ -1,8 +1,8 @@ use crate::cli::args::ZxyifyArgs; +use crate::errors::UtilesResult; use crate::mbt::zxyify::unzxyify; use crate::sqlite::AsyncSqliteConn; use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; -use crate::UtilesResult; #[tracing::instrument] pub async fn zxyify_main(args: ZxyifyArgs) -> UtilesResult<()> { diff --git a/crates/utiles/src/cli/entry.rs b/crates/utiles/src/cli/entry.rs index 65aeeb61..19151686 100644 --- a/crates/utiles/src/cli/entry.rs +++ b/crates/utiles/src/cli/entry.rs @@ -1,26 +1,50 @@ -use clap::Parser; -use tracing::{debug, error}; - use crate::cli::args::{Cli, Commands}; use crate::cli::commands::{ - about_main, bounding_tile_main, children_main, contains_main, copy_main, dev_main, - fmtstr_main, info_main, lint_main, metadata_main, metadata_set_main, - neighbors_main, parent_main, pmtileid_main, quadkey_main, rimraf_main, serve_main, - shapes_main, tilejson_main, tiles_main, touch_main, update_main, vacuum_main, - zxyify_main, + about_main, agg_hash_main, bounding_tile_main, children_main, contains_main, + copy_main, dev_main, fmtstr_main, info_main, lint_main, metadata_main, + metadata_set_main, neighbors_main, parent_main, pmtileid_main, quadkey_main, + rimraf_main, serve_main, shapes_main, tilejson_main, tiles_main, touch_main, + update_main, vacuum_main, zxyify_main, }; use crate::errors::UtilesResult; use crate::lager::{init_tracing, LogConfig}; use crate::signal::shutdown_signal; use crate::UtilesError; +use clap::{CommandFactory, FromArgMatches}; +use tracing::{debug, error}; +use utiles_core::VERSION; + +pub struct CliOpts { + pub argv: Option>, + pub clid: Option<&'static str>, +} + +impl Default for CliOpts { + fn default() -> Self { + Self { + argv: None, + clid: Option::from("rust"), + } + } +} -pub async fn cli_main( - argv: Option>, - // loop_fn: Option<&dyn Fn()>, -) -> UtilesResult { +impl CliOpts { + #[must_use] + pub fn aboot_str(&self) -> String { + format!( + "utiles cli ({}) ~ v{}", + self.clid.unwrap_or("rust"), + VERSION + ) + } +} + +pub async fn cli_main(cliops: Option) -> UtilesResult { tokio::select! { res = async { - cli_main_inner(argv).await + cli_main_inner( + cliops + ).await } => { debug!("Done. :)"); res @@ -39,12 +63,23 @@ pub async fn cli_main( } #[allow(clippy::unused_async)] -pub async fn cli_main_inner(argv: Option>) -> UtilesResult { +pub async fn cli_main_inner(cliopts: Option) -> UtilesResult { // print args - let argv = argv.unwrap_or_else(|| std::env::args().collect::>()); + let opts = cliopts.unwrap_or_default(); + let argv = opts.argv.unwrap_or_else(|| std::env::args().collect()); + let about_str = format!( + "utiles cli ({}) ~ v{}", + opts.clid.unwrap_or("rust"), + VERSION + ); // set caller if provided - let args = Cli::parse_from(&argv); + let cli = Cli::command().about(about_str); + let matches = cli.get_matches_from( + // argv.clone() + &argv, + ); + let args = Cli::from_arg_matches(&matches).expect("from_arg_matches failed"); // if the command is "dev" init tracing w/ debug let logcfg = if let Commands::Dev(_) = args.command { @@ -61,21 +96,24 @@ pub async fn cli_main_inner(argv: Option>) -> UtilesResult { } }; init_tracing(&logcfg)?; + debug!("args: {:?}", std::env::args().collect::>()); debug!("argv: {:?}", argv); debug!("args: {:?}", args); let res: UtilesResult<()> = match args.command { Commands::About => about_main(), + Commands::Db(dbcmds) => dbcmds.run().await, Commands::Lint(args) => lint_main(&args).await, - Commands::Touch(args) => touch_main(&args), - Commands::Vacuum(args) => vacuum_main(&args), + Commands::Touch(args) => touch_main(&args).await, + Commands::Vacuum(args) => vacuum_main(&args).await, Commands::Metadata(args) => metadata_main(&args).await, Commands::MetadataSet(args) => metadata_set_main(&args).await, Commands::Update(args) => update_main(&args).await, - Commands::Tilejson(args) => tilejson_main(&args), + Commands::Tilejson(args) => tilejson_main(&args).await, Commands::Copy(args) => copy_main(args).await, - Commands::Info(args) => info_main(&args), + Commands::Info(args) => info_main(&args).await, + Commands::AggHash(args) => agg_hash_main(&args).await, Commands::Dev(args) => dev_main(args).await, Commands::Rimraf(args) => rimraf_main(args).await, Commands::Contains { filepath, lnglat } => { @@ -88,7 +126,6 @@ pub async fn cli_main_inner(argv: Option>) -> UtilesResult { Commands::Pmtileid(args) => pmtileid_main(args), Commands::BoundingTile(args) => bounding_tile_main(args), Commands::Tiles(args) => tiles_main(args, None).await, - // Commands::Tiles(args) => sleep(args, None).await, Commands::Neighbors(args) => neighbors_main(args), Commands::Children(args) => children_main(args), Commands::Parent(args) => parent_main(args), @@ -108,17 +145,14 @@ pub async fn cli_main_inner(argv: Option>) -> UtilesResult { } // not sure why this is needed... cargo thinks it's unused??? -pub fn cli_main_sync( - argv: Option>, - // loop_fn: Option<&dyn Fn()>, -) -> UtilesResult { +pub fn cli_main_sync(opts: Option) -> UtilesResult { tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .expect( "tokio::runtime::Builder::new_multi_thread().enable_all().build() failed.", ) - .block_on(async { cli_main(argv).await }) + .block_on(async { cli_main(opts).await }) } #[cfg(test)] diff --git a/crates/utiles/src/cli/mod.rs b/crates/utiles/src/cli/mod.rs index bde4870a..e6c25118 100644 --- a/crates/utiles/src/cli/mod.rs +++ b/crates/utiles/src/cli/mod.rs @@ -14,5 +14,4 @@ mod stdin2string; mod stdinterator; mod stdinterator_filter; -pub use crate::cli::entry::cli_main; -pub use crate::cli::entry::cli_main_sync; +pub use crate::cli::entry::{cli_main, cli_main_sync, CliOpts}; diff --git a/crates/utiles/src/cli/stdin2string.rs b/crates/utiles/src/cli/stdin2string.rs index 2d1d0240..32675777 100644 --- a/crates/utiles/src/cli/stdin2string.rs +++ b/crates/utiles/src/cli/stdin2string.rs @@ -1,4 +1,4 @@ -use crate::UtilesResult; +use crate::errors::UtilesResult; use std::io::{self, Read}; pub fn stdin2string() -> UtilesResult { diff --git a/crates/utiles/src/config.rs b/crates/utiles/src/config.rs new file mode 100644 index 00000000..7e9da4f9 --- /dev/null +++ b/crates/utiles/src/config.rs @@ -0,0 +1,39 @@ +//! Utiles configuration +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub struct LintConfig { + pub include: Vec, + pub exclude: Vec, + + pub rules: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct UtilesConfig { + pub lint: LintConfig, + // TODO: server/log config + // pub log: LagerConfig, + // pub serve : ServeConfig, +} + +// #[derive(Debug, Deserialize, Serialize)] +// pub struct ServeConfig { +// pub host: String, +// pub port: u16, +// } +// +// #[derive(Debug, Deserialize, Serialize)] +// pub struct LagerConfig { +// pub level: String, +// pub json: bool, +// } + +// impl Default for LagerConfig { +// fn default() -> Self { +// Self { +// level: "info".to_string(), +// json: false, +// } +// } +// } diff --git a/crates/utiles/src/copy/cfg.rs b/crates/utiles/src/copy/cfg.rs index c20e963a..66f97e61 100644 --- a/crates/utiles/src/copy/cfg.rs +++ b/crates/utiles/src/copy/cfg.rs @@ -2,12 +2,14 @@ use std::path::PathBuf; use serde::Serialize; -use utiles_core::zoom::{ZoomOrZooms, ZoomSet}; -use utiles_core::{tile_ranges, BBox}; - -use crate::errors::{UtilesCopyError, UtilesResult}; -use crate::mbt::MbtType; +use crate::cli::args::CopyArgs; +use crate::errors::UtilesCopyError; +use crate::errors::UtilesResult; +use crate::mbt::hash_types::HashType; +use crate::mbt::{MbtType, TilesFilter}; use crate::sqlite::InsertStrategy; +use utiles_core::zoom::ZoomSet; +use utiles_core::BBox; #[derive(Debug, Clone, Serialize, Default)] pub struct CopyConfig { @@ -23,68 +25,40 @@ pub struct CopyConfig { pub jobs: Option, pub istrat: InsertStrategy, pub dbtype: Option, + pub hash: Option, +} + +impl From<&CopyArgs> for CopyConfig { + fn from(args: &CopyArgs) -> CopyConfig { + let dbtype = args.dbtype.as_ref().map(|dbtype| dbtype.into()); + CopyConfig { + src: PathBuf::from(&args.src), + dst: PathBuf::from(&args.dst), + zset: args.zoom_set(), + zooms: args.zooms(), + verbose: true, + bboxes: args.bboxes(), + bounds_string: args.bounds(), + force: false, + dryrun: false, + jobs: args.jobs, + istrat: InsertStrategy::from(args.conflict), + hash: args.hash, + dbtype, + } + } } impl CopyConfig { pub fn src_dbpath_str(&self) -> String { self.src.to_string_lossy().to_string() } - pub fn mbtiles_sql_where( - &self, - // zoom_levels: Option>, - ) -> UtilesResult { - let pred = match (&self.bboxes, &self.zooms) { - (Some(bbox), Some(zooms)) => { - // let zooms = self.zooms.unwrap_or(ZoomSet::all().into()); - let zboxes = bbox - .iter() - .flat_map(|b| { - tile_ranges(b.tuple(), ZoomOrZooms::Zooms(zooms.clone())) - }) - .collect::>(); - let pred = zboxes - .iter() - .map(utiles_core::tile_zbox::TileZBoxes::mbtiles_sql_where) - .collect::>() - .join(" OR "); - format!("({pred})") - } - (Some(bbox), None) => { - let zboxes = bbox - .iter() - .flat_map(|b| { - tile_ranges( - b.tuple(), - ZoomOrZooms::Zooms(ZoomSet::all().into()), - ) - }) - .collect::>(); - let pred = zboxes - .iter() - .map(utiles_core::tile_zbox::TileZBoxes::mbtiles_sql_where) - .collect::>() - .join(" OR "); - format!("({pred})") - } - (None, Some(zooms)) => { - format!( - "zoom_level IN ({zooms})", - zooms = zooms - .iter() - .map(std::string::ToString::to_string) - .collect::>() - .join(",") - ) - } - (None, None) => String::new(), - }; - // attach 'WHERE' - if pred.is_empty() { - Ok(pred) - } else { - Ok(format!("WHERE {pred}")) - } + + pub fn mbtiles_sql_where(&self) -> UtilesResult { + let tf = TilesFilter::new(self.bboxes.clone(), self.zooms.clone()); + tf.mbtiles_sql_where(None) } + pub fn check_src_dst_same(&self) -> UtilesResult<()> { if self.src == self.dst { Err( @@ -125,79 +99,3 @@ impl CopyConfig { } } } - -// -// impl crate::cli::commands::copy::CopyConfigV1 { -// pub fn new( -// src: crate::cli::commands::copy::Source, -// dst: crate::cli::commands::copy::Destination, -// zooms: Option>, -// bbox: Option, -// ) -> Self { -// Self { -// src, -// dst, -// zooms, -// bbox, -// } -// } -// -// // pub fn sql_where_for_zoom(&self, zoom: u8) -> String { -// // let pred = match &self.bbox { -// // Some(bbox) => { -// // let trange = tile_ranges(bbox.tuple(), vec![zoom].into()); -// // trange.sql_where(Some(true)) -// // } -// // None => { -// // format!("zoom_level = {zoom}") -// // } -// // }; -// // // attach 'WHERE' -// // if pred.is_empty() { -// // pred -// // } else { -// // format!("WHERE {pred}") -// // } -// // } -// -// pub fn mbtiles_sql_where( -// &self, -// zoom_levels: Option>, -// ) -> UtilesResult { -// let pred = match (&self.bbox, &self.zooms) { -// (Some(bbox), Some(zooms)) => { -// let trange = tile_ranges( -// bbox.tuple(), -// zoom_levels.unwrap_or(zooms.clone()).into(), -// )?; -// trange.mbtiles_sql_where() -// } -// (Some(bbox), None) => { -// let trange = tile_ranges( -// bbox.tuple(), -// zoom_levels -// .unwrap_or((0..28).map(|z| z as u8).collect::>()) -// .into(), -// )?; -// trange.mbtiles_sql_where() -// } -// (None, Some(zooms)) => { -// format!( -// "zoom_level IN ({zooms})", -// zooms = zooms -// .iter() -// .map(std::string::ToString::to_string) -// .collect::>() -// .join(",") -// ) -// } -// (None, None) => String::new(), -// }; -// // attach 'WHERE' -// if pred.is_empty() { -// Ok(pred) -// } else { -// Ok(format!("WHERE {pred}")) -// } -// } -// } diff --git a/crates/utiles/src/copy/mod.rs b/crates/utiles/src/copy/mod.rs index 5bbbe218..3a213f1b 100644 --- a/crates/utiles/src/copy/mod.rs +++ b/crates/utiles/src/copy/mod.rs @@ -70,14 +70,12 @@ pub async fn copy(cfg: &CopyConfig) -> UtilesResult<()> { // TODO: figure out what I was doing here there is some duplication // of things happening... // make sure input file exists and is file... - let src = - get_tile_src(pasta.cfg.src.to_str().ok_or_else(|| { - UtilesError::Error("src is not a valid string".to_string()) - })?)?; - let dst = - get_tile_dst(pasta.cfg.dst.to_str().ok_or_else(|| { - UtilesError::Error("dst is not a valid string".to_string()) - })?)?; + let src = get_tile_src(pasta.cfg.src.to_str().ok_or_else(|| { + UtilesError::PathConversionError(pasta.cfg.src.to_string_lossy().to_string()) + })?)?; + let dst = get_tile_dst(pasta.cfg.dst.to_str().ok_or_else(|| { + UtilesError::PathConversionError(pasta.cfg.dst.to_string_lossy().to_string()) + })?)?; let srcdst = match (src, dst) { (Source::Mbtiles(_src), Destination::Fs(_dst)) => Ok(CopySrcDest::Mbtiles2Fs), diff --git a/crates/utiles/src/copy/pasta.rs b/crates/utiles/src/copy/pasta.rs index 357844b6..304ddac7 100644 --- a/crates/utiles/src/copy/pasta.rs +++ b/crates/utiles/src/copy/pasta.rs @@ -1,13 +1,16 @@ #![allow(dead_code)] +use tracing::{debug, info, warn}; + +use utiles_core::UtilesCoreError; + use crate::copy::CopyConfig; -use crate::errors::{UtilesCopyError, UtilesResult}; +use crate::errors::UtilesCopyError; +use crate::errors::UtilesResult; use crate::mbt::MbtType; use crate::sqlite::{AsyncSqliteConn, Sqlike3Async}; use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use crate::UtilesError; -use tracing::{debug, warn}; -use utiles_core::UtilesCoreError; #[derive(Debug)] pub struct CopyPasta { @@ -35,7 +38,9 @@ impl CopyPasta { } /// Returns the destination db and a bool indicating if it was created - pub async fn get_dst_db(&self) -> UtilesResult<(MbtilesAsyncSqliteClient, bool)> { + pub async fn get_dst_db( + &self, + ) -> UtilesResult<(MbtilesAsyncSqliteClient, bool, MbtType)> { // if the dst is a file... we gotta get it... let dst_db_res = MbtilesAsyncSqliteClient::open_existing(&self.cfg.dst).await; let (dst_db, is_new) = match dst_db_res { @@ -55,7 +60,8 @@ impl CopyPasta { } }; dst_db.register_utiles_sqlite_functions().await?; - Ok((dst_db, is_new)) + let db_type_queried = dst_db.query_mbt_type().await?; + Ok((dst_db, is_new, db_type_queried)) } pub async fn copy_metadata( @@ -117,12 +123,12 @@ impl CopyPasta { let src_db_name = "src"; let where_clause = self.cfg.mbtiles_sql_where()?; let insert_strat = self.cfg.istrat.sql_prefix().to_string(); - let hash_type = "md5_hex"; - + let hash_type_fn_string = + self.cfg.hash.unwrap_or_default().sqlite_hex_fn_name(); let n_tiles_inserted = dst_db.conn( move |x| { let insert_statement = &format!( - "{insert_strat} INTO tiles_with_hash (zoom_level, tile_column, tile_row, tile_data, tile_hash) SELECT zoom_level, tile_column, tile_row, tile_data, {hash_type}(tile_data) as tile_hash FROM {src_db_name}.tiles {where_clause}" + "{insert_strat} INTO tiles_with_hash (zoom_level, tile_column, tile_row, tile_data, tile_hash) SELECT zoom_level, tile_column, tile_row, tile_data, {hash_type_fn_string}(tile_data) as tile_hash FROM {src_db_name}.tiles {where_clause}" ); debug!("Executing tiles insert: {:?}", insert_statement); x.execute( @@ -147,36 +153,50 @@ impl CopyPasta { let src_db_name = "src"; let where_clause = self.cfg.mbtiles_sql_where()?; let insert_strat = self.cfg.istrat.sql_prefix().to_string(); - let hash_type = "md5_hex"; + let hash_type_fn_string = + self.cfg.hash.unwrap_or_default().sqlite_hex_fn_name(); + debug!("hash fn: {}", hash_type_fn_string); - let n_tiles_inserted = dst_db.conn( - move |x| { + let n_tiles_inserted = dst_db + .conn(move |x| { let insert_statement = &format!( - "{insert_strat} INTO map (zoom_level, tile_column, tile_row, tile_id) SELECT zoom_level, tile_column, tile_row, {hash_type}(tile_data) as tile_id FROM {src_db_name}.tiles {where_clause}" + " +{insert_strat} INTO map (zoom_level, tile_column, tile_row, tile_id) +SELECT + zoom_level as zoom_level, + tile_column as tile_column, + tile_row as tile_row, + {hash_type_fn_string}(tile_data) AS tile_id +FROM + {src_db_name}.tiles +{where_clause}; +" ); debug!("Executing tiles insert: {:?}", insert_statement); - let changes = x.execute( - insert_statement, - [], - )?; + let changes = x.execute(insert_statement, [])?; // now just join and insert the images... let insert_statement = &format!( " - INSERT OR IGNORE INTO images (tile_id, tile_data) - SELECT map.tile_id, tiles.tile_data - FROM map - JOIN {src_db_name}.tiles ON map.zoom_level = {src_db_name}.tiles.zoom_level AND map.tile_column = {src_db_name}.tiles.tile_column AND map.tile_row = {src_db_name}.tiles.tile_row +INSERT OR IGNORE INTO images (tile_id, tile_data) +SELECT + map.tile_id, + tiles.tile_data +FROM + map +JOIN + {src_db_name}.tiles +ON + map.zoom_level = {src_db_name}.tiles.zoom_level + AND map.tile_column = {src_db_name}.tiles.tile_column + AND map.tile_row = {src_db_name}.tiles.tile_row; " ); debug!("Executing images insert: {:?}", insert_statement); - let changes2 = x.execute( - insert_statement, - [], - )?; + let changes2 = x.execute(insert_statement, [])?; Ok(changes + changes2) - } - ).await?; + }) + .await?; if n_tiles_inserted == 0 { warn!("No tiles inserted!"); @@ -206,17 +226,11 @@ impl CopyPasta { // do the thing debug!("Copying tiles from src to dst: {:?}", self.cfg); self.copy_tiles_zbox_hash(dst_db).await - // self.copy_tiles_normal(dst_db, src_db_name, &where_clause, &insert_strat).await - // let emsg = format!("Unsupported/unimplemented db-type {:?}", dst_db_type); - // Err(UtilesCoreError::Unimplemented(emsg).into()) } MbtType::Norm => { // do the thing debug!("Copying tiles from src to dst: {:?}", self.cfg); self.copy_tiles_zbox_norm(dst_db).await - // self.copy_tiles_normal(dst_db, src_db_name, &where_clause, &insert_strat).await - // let emsg = format!("Unsupported/unimplemented db-type {:?}", dst_db_type); - // Err(UtilesCoreError::Unimplemented(emsg).into()) } _ => { // do the thing @@ -245,28 +259,35 @@ impl CopyPasta { // join on zoom_level, tile_column, tile_row for src and dst and // see if there is any overlap + let where_clause = self.cfg.mbtiles_sql_where()?; // TODO: check if minzoom and maxzoom overlap - let has_conflict = dst_db.conn( - move |c| { + let has_conflict = dst_db + .conn(move |c| { let src_db_name = "src"; let check_statement = &format!( - "SELECT COUNT(*) FROM tiles JOIN {src_db_name}.tiles ON main.tiles.zoom_level = {src_db_name}.tiles.zoom_level AND main.tiles.tile_column = {src_db_name}.tiles.tile_column AND main.tiles.tile_row = {src_db_name}.tiles.tile_row LIMIT 1" + r" +SELECT COUNT(*) +FROM ( + SELECT main.tiles.zoom_level, main.tiles.tile_column, main.tiles.tile_row + FROM main.tiles + {where_clause} +) AS filtered_tiles +JOIN {src_db_name}.tiles ON + filtered_tiles.zoom_level = {src_db_name}.tiles.zoom_level + AND filtered_tiles.tile_column = {src_db_name}.tiles.tile_column + AND filtered_tiles.tile_row = {src_db_name}.tiles.tile_row +LIMIT 1; + " ); - debug!("Executing check_statement: {:?}", check_statement); - c.query_row( - check_statement, - [], - |row| { - let r: i64 = row.get(0)?; - - Ok(r) - }, - ) - } - ).await.map_err( - UtilesError::AsyncSqliteError - )?; + c.query_row(check_statement, [], |row| { + let r: i64 = row.get(0)?; + + Ok(r) + }) + }) + .await + .map_err(UtilesError::AsyncSqliteError)?; Ok(has_conflict > 0) } @@ -275,7 +296,7 @@ impl CopyPasta { // doing preflight check debug!("Preflight check"); self.preflight_check()?; - let (dst_db, is_new) = self.get_dst_db().await?; + let (dst_db, is_new, db_type) = self.get_dst_db().await?; let src_db_name = "src"; @@ -285,6 +306,7 @@ impl CopyPasta { dst_db.attach_db(&src_db_path, src_db_name).await?; debug!("OK: Attached src db"); if !is_new && self.cfg.istrat.requires_check() { + info!("No conflict strategy provided; checking for conflict"); let has_conflict = self.check_conflict(&dst_db).await?; if has_conflict { warn!("Conflict detected!"); @@ -293,8 +315,15 @@ impl CopyPasta { ) .into()); } + } else if is_new { + debug!("dst db is new; not checking for conflict"); + } else { + debug!( + "No check required for conflict strategy: {}", + self.cfg.istrat.to_string() + ); } - + info!("Copying tiles: {:?} -> {:?}", self.cfg.src, self.cfg.dst); let n_tiles_inserted = self.copy_tiles_zbox(&dst_db).await?; debug!("n_tiles_inserted: {:?}", n_tiles_inserted); @@ -303,6 +332,21 @@ impl CopyPasta { debug!("n_metadata_inserted: {:?}", n_metadata_inserted); } + dst_db.metadata_set("dbtype", db_type.as_str()).await?; + if db_type == MbtType::Hash || db_type == MbtType::Norm { + dst_db + .metadata_set( + "tileid", + self.cfg + .hash + .unwrap_or_default() + .to_string() + .to_string() + .as_str(), + ) + .await?; + } + debug!("Detaching src db..."); dst_db.detach_db(src_db_name).await?; debug!("Detached src db!"); diff --git a/crates/utiles/src/copy/pyramid.rs b/crates/utiles/src/copy/pyramid.rs index db87304a..6684e894 100644 --- a/crates/utiles/src/copy/pyramid.rs +++ b/crates/utiles/src/copy/pyramid.rs @@ -9,7 +9,8 @@ use tracing::{debug, info, warn}; use utiles_core::TileLike; use crate::copy::CopyConfig; -use crate::errors::{UtilesError, UtilesResult}; +use crate::errors::UtilesError; +use crate::errors::UtilesResult; use crate::mbt::MbtTileRow; use crate::utilesqlite::Mbtiles; diff --git a/crates/utiles/src/errors.rs b/crates/utiles/src/errors.rs index c7925191..d3345177 100644 --- a/crates/utiles/src/errors.rs +++ b/crates/utiles/src/errors.rs @@ -1,6 +1,5 @@ use thiserror::Error; -pub type UtilesResult = Result; #[derive(Error, Debug)] pub enum UtilesCopyError { #[error("src and dst: {0}")] @@ -24,15 +23,24 @@ pub enum UtilesError { #[error("No fspath extension: {0}")] NoFspathExtension(String), + #[error("No fspath stem: {0}")] + NoFspathStem(String), + #[error("File does not exist: {0}")] FileDoesNotExist(String), + #[error("metadata error: {0}")] + MetadataError(String), + #[error("Path already exists: {0}")] PathExistsError(String), #[error("Not a file: {0}")] NotAFile(String), + #[error("Non mbtiles sqlite db: {0}")] + NonMbtilesSqliteDb(String), + #[error("parse int error: {0}")] ParseIntError(#[from] std::num::ParseIntError), @@ -93,3 +101,5 @@ pub enum UtilesError { #[error("json_patch error: {0}")] JsonPatchError(#[from] json_patch::PatchError), } + +pub type UtilesResult = Result; diff --git a/crates/utiles/src/fs_async.rs b/crates/utiles/src/fs_async.rs new file mode 100644 index 00000000..b0b063b2 --- /dev/null +++ b/crates/utiles/src/fs_async.rs @@ -0,0 +1,34 @@ +use tokio::fs; + +use crate::errors::UtilesResult; +use crate::UtilesError; + +pub async fn file_exists>(p: P) -> bool { + let metadata = fs::metadata(p).await; + match metadata { + Ok(metadata) => metadata.is_file(), + Err(_) => false, + } +} + +// pub async fn dir_exists>(p: P) -> bool { +// let metadata = fs::metadata(p).await; +// match metadata { +// Ok(metadata) => metadata.is_dir(), +// Err(_) => false, +// } +// } + +pub async fn file_exists_err>(p: P) -> UtilesResult { + if file_exists(&p).await { + Ok(true) + } else { + let p_str = p.as_ref().to_string_lossy(); + Err(UtilesError::FileDoesNotExist(format!("{p_str:?}"))) + } +} + +pub async fn filesize_async>(p: P) -> Option { + let metadata = fs::metadata(p).await.ok()?; + Some(metadata.len()) +} diff --git a/crates/utiles/src/lager.rs b/crates/utiles/src/lager.rs index 77eac9d3..a72961dc 100644 --- a/crates/utiles/src/lager.rs +++ b/crates/utiles/src/lager.rs @@ -49,5 +49,6 @@ pub fn init_tracing(log_config: &LogConfig) -> UtilesResult<()> { } } } + debug!("lager-config: {:?}", log_config); Ok(()) } diff --git a/crates/utiles/src/lib.rs b/crates/utiles/src/lib.rs index 9c1f25cd..d91554a1 100644 --- a/crates/utiles/src/lib.rs +++ b/crates/utiles/src/lib.rs @@ -14,21 +14,25 @@ #![deny(clippy::pedantic)] #![allow(clippy::redundant_closure_for_method_calls)] #![allow(clippy::missing_errors_doc)] -#![allow(clippy::missing_panics_doc)] +// #![allow(clippy::missing_panics_doc)] #![allow(clippy::module_name_repetitions)] #![allow(clippy::unnecessary_wraps)] #![allow(clippy::cast_possible_truncation)] #![allow(clippy::cast_sign_loss)] #![allow(clippy::cast_possible_wrap)] -pub mod core; pub use core::*; -pub use errors::{UtilesError, UtilesResult}; +pub use errors::UtilesError; +pub use errors::UtilesResult; +pub use lager::init_tracing; pub use tile_strfmt::TileStringFormatter; pub mod cli; +mod config; mod copy; +pub mod core; pub mod dev; pub(crate) mod errors; +mod fs_async; pub mod gj; mod globster; mod lager; @@ -43,145 +47,7 @@ mod tile_strfmt; pub mod utilejson; pub mod utilesqlite; -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); - -/// Tile macro to create a new tile from x, y, z -#[macro_export] -macro_rules! utile { - ($x:expr, $y:expr, $z:expr) => { - Tile::new($x, $y, $z) - }; -} - -/// macro to create a new point. -/// Replacement for coord! macro from geo-types -/// -/// # Examples -/// -/// ``` -/// use utiles::{point2d, Point2d}; -/// let p = point2d!{ x: 1.0, y: 2.0 }; -/// assert_eq!(p.x(), 1.0); -/// assert_eq!(p.y(), 2.0); -/// ``` -#[macro_export] -macro_rules! point2d { - { x: $x:expr, y: $y:expr } => { - Point2d::new($x, $y) - }; -} - #[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used)] - - use std::collections::HashSet; - - use super::*; - - #[test] - fn zoom_or_zooms() { - let z = as_zooms(1.into()); - assert_eq!(z, vec![1]); - let z = as_zooms(vec![1, 2, 3].into()); - assert_eq!(z, vec![1, 2, 3]); - } - - #[test] - fn tiles_generator() { - let bounds = (-105.0, 39.99, -104.99, 40.0); - let tiles = tiles(bounds, vec![14].into()); - let expect = vec![Tile::new(3413, 6202, 14), Tile::new(3413, 6203, 14)]; - assert_eq!(tiles.collect::>(), expect); - } +mod tests; - #[test] - fn tiles_single_zoom() { - let bounds = (-105.0, 39.99, -104.99, 40.0); - let tiles = tiles(bounds, 14.into()); - let expect = vec![Tile::new(3413, 6202, 14), Tile::new(3413, 6203, 14)]; - assert_eq!(tiles.collect::>(), expect); - - let num_tiles = tiles_count(bounds, 14.into()).unwrap(); - assert_eq!(num_tiles, 2); - } - - #[test] - fn tiles_anti_meridian() { - let bounds = (175.0, 5.0, -175.0, 10.0); - let mut tiles: Vec = tiles(bounds, 2.into()).collect(); - tiles.sort(); - let mut expected = vec![Tile::new(3, 1, 2), Tile::new(0, 1, 2)]; - expected.sort(); - assert_eq!(tiles, expected); - } - - #[test] - fn tile_is_valid() { - let valid_tiles = vec![ - Tile::new(0, 0, 0), - Tile::new(0, 0, 1), - Tile::new(1, 1, 1), - Tile::new(243, 166, 9), - ]; - - for tile in valid_tiles { - assert!(tile.valid(), "{tile:?} is not valid"); - } - } - - #[test] - fn tile_is_invalid() { - let invalid_tiles = vec![ - Tile::new(0, 1, 0), - Tile::new(1, 0, 0), - Tile::new(1, 1, 0), - Tile::new(1, 234, 1), - ]; - - for tile in invalid_tiles { - assert!(!tile.valid(), "{tile:?} is valid"); - } - } - - #[test] - fn test_macro() { - let tile = utile!(0, 0, 0); - assert_eq!(tile, Tile::new(0, 0, 0)); - } - - #[test] - fn test_simplify() { - let children = utile!(243, 166, 9).children(Some(12)); - assert_eq!(children.len(), 64); - let mut children = children.into_iter().collect::>(); - children.truncate(61); - children.push(children[0]); - let simplified = simplify(children.into_iter().collect::>()); - let targets = vec![ - utile!(487, 332, 10), - utile!(486, 332, 10), - utile!(487, 333, 10), - utile!(973, 667, 11), - utile!(973, 666, 11), - utile!(972, 666, 11), - utile!(1944, 1334, 12), - ]; - for target in targets { - assert!(simplified.contains(&target)); - } - } - - #[test] - fn test_simplify_removal() { - let tiles = vec![ - utile!(1298, 3129, 13), - utile!(649, 1564, 12), - utile!(650, 1564, 12), - ]; - let simplified = simplify(tiles.into_iter().collect::>()); - assert!(!simplified.contains(&utile!(1298, 3129, 13))); - assert!(simplified.contains(&utile!(650, 1564, 12))); - assert!(simplified.contains(&utile!(649, 1564, 12))); - } -} +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/utiles/src/lint.rs b/crates/utiles/src/lint.rs deleted file mode 100644 index 6f173607..00000000 --- a/crates/utiles/src/lint.rs +++ /dev/null @@ -1,263 +0,0 @@ -use std::path::{Path, PathBuf}; - -use colored::Colorize; -use thiserror::Error; - -use crate::mbt::metadata2duplicates; -use crate::sqlite::AsyncSqliteConn; -use crate::utilesqlite::mbtiles::{ - has_unique_index_on_metadata, metadata_table_name_is_primary_key, -}; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; -use crate::{utilesqlite, UtilesError}; - -pub const REQUIRED_METADATA_FIELDS: [&str; 5] = - ["bounds", "format", "maxzoom", "minzoom", "name"]; -// pub const RECCOMENDED_METADATA_FIELDS: [&str; 4] = ["center", "description", "version", "attribution", "type"]; - -#[derive(Error, Debug)] -pub enum UtilesLintWarning { - #[error("missing mbtiles magic-number/application_id")] - MbtMissingMagicNumber, -} - -#[derive(Error, Debug)] -pub enum UtilesLintError { - #[error("unable to open: {0}")] - UnableToOpen(String), - - #[error("not a sqlite database error: {0}")] - NotASqliteDb(String), - - #[error("not a mbtiles database error: {0}")] - NotAMbtilesDb(String), - - #[error("no tiles table/view")] - MbtMissingTiles, - - #[error("no metadata table/view")] - MbtMissingMetadata, - - #[error("missing mbtiles magic-number/application_id")] - MbtMissingMagicNumber, - - #[error("Unrecognized mbtiles magic-number/application_id: {0} != 0x4d504258")] - MbtUnknownMagicNumber(u32), - - #[error("missing index: {0}")] - MissingIndex(String), - - #[error("missing unique index: {0}")] - MissingUniqueIndex(String), - - #[error("duplicate metadata key: {0}")] - DuplicateMetadataKey(String), - - #[error("metadata k/v missing: {0}")] - MbtMissingMetadataKv(String), - - #[error("unknown error: {0}")] - Unknown(String), - - #[error("lint errors {0:?}")] - LintErrors(Vec), - - #[error("utiles error: {0}")] - UtilesError(#[from] UtilesError), -} - -impl UtilesLintError { - #[must_use] - pub fn format_error(&self, filepath: &str) -> String { - let errcode = "MBT".red(); - let errstr = format!("{errcode}: {self}"); - - // let error_str = format!("STTUFF -- {}", self.to_string()); - let e_str = format!("{filepath}: {errstr}"); - e_str - // match self { - // UtilesError::CoreError(e) => e.to_string(), - // UtilesError::Unimplemented(e) => e.to_string(), - // UtilesError::SqliteError(e) => e.to_string(), - // UtilesError::AsyncSqliteError(e) => e.to_string(), - // UtilesError::FileDoesNotExist(e) => e.to_string(), - // UtilesError::ParseIntError(e) => e.to_string(), - // UtilesError::Error(e) => e.to_string(), - // UtilesError::Unknown(e) => e.to_string(), - // } - } -} - -// impl From for UtilesLintError { -// fn from(e: UtilesError) -> Self { -// UtilesLintError::UtilesError(e) -// } -// } - -// combination of all errors and warnings -#[derive(Error, Debug)] -pub enum UtilesLint { - #[error("error: {0}")] - Error(#[from] UtilesLintError), - - #[error("warning: {0}")] - Warning(#[from] UtilesLintWarning), - - #[error("lint errors {0:?}")] - Errors(Vec), - - #[error("lint warnings {0:?}")] - Warnings(Vec), -} - -pub type UtilesLintResult = Result; - -#[derive(Debug)] -pub struct MbtilesLinter { - pub path: PathBuf, - pub fix: bool, -} - -impl MbtilesLinter { - #[must_use] - pub fn new>(path: T, fix: bool) -> Self { - MbtilesLinter { - path: path.as_ref().to_path_buf(), - fix, - } - } - async fn open_mbtiles( - &self, - ) -> UtilesLintResult { - let pth = self - .path - .to_str() - .ok_or(UtilesLintError::Unknown("unknown path".to_string()))?; - let mbtiles = - match utilesqlite::MbtilesAsyncSqliteClient::open_readonly(pth).await { - Ok(m) => m, - Err(e) => { - return Err(UtilesLintError::UnableToOpen(e.to_string())); - } - }; - Ok(mbtiles) - } - - pub async fn check_magic_number( - mbt: &MbtilesAsyncSqliteClient, - ) -> UtilesLintResult<()> { - let magic_number_res = mbt.magic_number().await; - match magic_number_res { - Ok(magic_number) => { - if magic_number == 0x4d50_4258 { - Ok(()) - } else if magic_number == 0 { - Err(UtilesLintError::MbtMissingMagicNumber) - } else { - Err(UtilesLintError::MbtUnknownMagicNumber(magic_number)) - } - } - Err(e) => Err(e.into()), - } - } - pub async fn check_metadata_rows( - mbt: &MbtilesAsyncSqliteClient, - ) -> UtilesLintResult<()> { - let metadata_rows = mbt.metadata_rows().await?; - let metadata_keys = metadata_rows - .iter() - .map(|r| r.name.clone()) - .collect::>(); - let missing_metadata_keys = REQUIRED_METADATA_FIELDS - .iter() - .filter(|k| !metadata_keys.contains(&(**k).to_string())) - .map(|k| (*k).to_string()) - .collect::>(); - if missing_metadata_keys.is_empty() { - Ok(()) - } else { - let errs = missing_metadata_keys - .iter() - .map(|k| UtilesLintError::MbtMissingMetadataKv(k.clone())) - .collect::>(); - Err(UtilesLintError::LintErrors(errs)) - } - } - - pub async fn check_metadata( - mbt: &MbtilesAsyncSqliteClient, - ) -> UtilesLintResult<()> { - // that metadata table exists - let has_unique_index_on_metadata_name = mbt - .conn(has_unique_index_on_metadata) - .await - .map_err(UtilesError::AsyncSqliteError)?; - - let mut errs = vec![]; - let name_is_pk = mbt - .conn(metadata_table_name_is_primary_key) - .await - .map_err(UtilesError::AsyncSqliteError)?; - if has_unique_index_on_metadata_name || name_is_pk { - let rows = mbt.metadata_rows().await?; - let duplicate_rows = metadata2duplicates(rows.clone()); - if !duplicate_rows.is_empty() { - errs.extend( - duplicate_rows - .keys() - .map(|k| UtilesLintError::DuplicateMetadataKey(k.clone())) - .collect::>(), - ); - } - } else { - errs.push(UtilesLintError::MissingUniqueIndex( - "metadata.name".to_string(), - )); - } - - let rows_errs = MbtilesLinter::check_metadata_rows(mbt).await; - if let Err(e) = rows_errs { - match e { - UtilesLintError::LintErrors(es) => { - errs.extend(es); - } - _ => { - errs.push(e); - } - } - } - if errs.is_empty() { - Ok(()) - } else { - Err(UtilesLintError::LintErrors(errs)) - } - } - - pub async fn lint(&self) -> UtilesLintResult> { - let mbt = self.open_mbtiles().await?; - if !mbt.is_mbtiles().await? { - let pth = self.path.to_str().unwrap_or("unknown-path"); - return Err(UtilesLintError::NotAMbtilesDb(pth.to_string())); - } - let mut lint_results = vec![]; - // lint_results.push(MbtilesLinter::check_magic_number(&mbt).await); - - match MbtilesLinter::check_metadata(&mbt).await { - Ok(()) => {} - Err(e) => match e { - UtilesLintError::LintErrors(errs) => { - lint_results.push(Ok(())); - lint_results.extend(errs.into_iter().map(Err)); - } - _ => { - lint_results.push(Err(e)); - } - }, - } - - Ok(lint_results - .into_iter() - .filter_map(std::result::Result::err) - .collect()) - } -} diff --git a/crates/utiles/src/lint/mbt_linter.rs b/crates/utiles/src/lint/mbt_linter.rs new file mode 100644 index 00000000..eadf9fc7 --- /dev/null +++ b/crates/utiles/src/lint/mbt_linter.rs @@ -0,0 +1,134 @@ +use crate::errors::UtilesResult; +use crate::lint::UtilesLintError; +use crate::mbt::metadata2duplicates; +use crate::sqlite::AsyncSqliteConn; +use crate::utilesqlite::mbtiles::{ + has_unique_index_on_metadata, metadata_table_name_is_primary_key, +}; +use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::{utilesqlite, UtilesError}; +use std::path::{Path, PathBuf}; +use tracing::warn; + +#[derive(Debug)] +pub struct MbtilesLinter { + pub path: PathBuf, + pub fix: bool, +} + +impl MbtilesLinter { + #[must_use] + pub fn new>(path: T, fix: bool) -> Self { + MbtilesLinter { + path: path.as_ref().to_path_buf(), + fix, + } + } + async fn open_mbtiles( + &self, + ) -> UtilesResult { + let pth = self.path.to_str().map_or_else( + || Err(UtilesError::PathConversionError("path".to_string())), + Ok, + )?; + let mbtiles = utilesqlite::MbtilesAsyncSqliteClient::open_readonly(pth).await?; + Ok(mbtiles) + } + + pub async fn check_magic_number( + mbt: &MbtilesAsyncSqliteClient, + ) -> UtilesResult> { + let magic_number_res = mbt.magic_number().await; + match magic_number_res { + Ok(magic_number) => { + if magic_number == 0x4d50_4258 { + Ok(None) + } else if magic_number == 0 { + Ok(Some(crate::lint::UtilesLintError::MbtMissingMagicNumber)) + } else { + Ok(Some(crate::lint::UtilesLintError::MbtUnknownMagicNumber( + magic_number, + ))) + } + } + Err(e) => Err(e), + } + } + pub async fn check_metadata_rows( + mbt: &MbtilesAsyncSqliteClient, + ) -> UtilesResult> { + let metadata_rows = mbt.metadata_rows().await?; + let metadata_keys = metadata_rows + .iter() + .map(|r| r.name.clone()) + .collect::>(); + let missing_metadata_keys = crate::lint::REQUIRED_METADATA_FIELDS + .iter() + .filter(|k| !metadata_keys.contains(&(**k).to_string())) + .map(|k| (*k).to_string()) + .collect::>(); + let errs = missing_metadata_keys + .iter() + .map(|k| crate::lint::UtilesLintError::MbtMissingMetadataKv(k.clone())) + .collect::>(); + Ok(errs) + } + + pub async fn check_metadata( + mbt: &MbtilesAsyncSqliteClient, + ) -> UtilesResult> { + // that metadata table exists + let has_unique_index_on_metadata_name = mbt + .conn(has_unique_index_on_metadata) + .await + .map_err(UtilesError::AsyncSqliteError)?; + + let mut errs = vec![]; + let name_is_pk = mbt + .conn(metadata_table_name_is_primary_key) + .await + .map_err(UtilesError::AsyncSqliteError)?; + if has_unique_index_on_metadata_name || name_is_pk { + let rows = mbt.metadata_rows().await?; + let duplicate_rows = metadata2duplicates(rows.clone()); + if !duplicate_rows.is_empty() { + errs.extend( + duplicate_rows + .keys() + .map(|k| UtilesLintError::DuplicateMetadataKey(k.clone())) + .collect::>(), + ); + } + } else { + errs.push(UtilesLintError::MissingUniqueIndex( + "metadata.name".to_string(), + )); + } + + let rows_errors = MbtilesLinter::check_metadata_rows(mbt).await?; + errs.extend(rows_errors); + Ok(errs) + } + + pub async fn lint(&self) -> UtilesResult> { + if self.fix { + warn!("Fix not implemented (yet)"); + } + let mbt = self.open_mbtiles().await?; + if !mbt.is_mbtiles_like().await? { + let pth = self.path.to_str().unwrap_or("unknown-path"); + return Err(UtilesError::NonMbtilesSqliteDb(pth.to_string())); + } + let mut lint_results = vec![]; + let magic_res = MbtilesLinter::check_magic_number(&mbt).await?; + if let Some(e) = magic_res { + lint_results.push(e); + } + let metadata_res = MbtilesLinter::check_metadata(&mbt).await?; + lint_results.extend(metadata_res); + let lint_errors = lint_results + .into_iter() + .collect::>(); + Ok(lint_errors) + } +} diff --git a/crates/utiles/src/lint/mod.rs b/crates/utiles/src/lint/mod.rs new file mode 100644 index 00000000..76b46183 --- /dev/null +++ b/crates/utiles/src/lint/mod.rs @@ -0,0 +1,140 @@ +use std::path::PathBuf; +use std::time::Duration; + +use colored::Colorize; +use futures::{stream, Stream, StreamExt}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tracing::{debug, warn}; + +use mbt_linter::MbtilesLinter; + +use crate::errors::UtilesResult; + +mod mbt_linter; + +pub const REQUIRED_METADATA_FIELDS: [&str; 5] = + ["bounds", "format", "maxzoom", "minzoom", "name"]; + +#[derive(Error, Debug, Clone, Deserialize, Serialize)] +pub enum UtilesLintError { + #[error("not a sqlite database error: {0}")] + NotASqliteDb(String), + + #[error("not a mbtiles database error: {0}")] + NotAMbtilesDb(String), + + #[error("no tiles table/view")] + MbtMissingTiles, + + #[error("no metadata table/view")] + MbtMissingMetadata, + + #[error("missing mbtiles magic-number/application_id")] + MbtMissingMagicNumber, + + #[error("Unrecognized mbtiles magic-number/application_id: {0} != 0x4d504258")] + MbtUnknownMagicNumber(u32), + + #[error("missing index: {0}")] + MissingIndex(String), + + #[error("missing unique index: {0}")] + MissingUniqueIndex(String), + + #[error("duplicate metadata key: {0}")] + DuplicateMetadataKey(String), + + #[error("metadata k/v missing: {0}")] + MbtMissingMetadataKv(String), + + #[error("lint errors {0:?}")] + LintErrors(Vec), +} + +impl UtilesLintError { + #[must_use] + pub fn format_error(&self, filepath: &str) -> String { + let errcode = "MBT".red(); + let errstr = format!("{errcode}: {self}"); + let e_str = format!("{filepath}: {errstr}"); + e_str + } +} +pub type UtilesLintResult = Result; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct FileLintResults { + fspath: String, + errors: Option>, + dt: Duration, +} +pub fn lint_filepaths_stream( + fspaths: &Vec, + fix: bool, +) -> impl Stream + '_ { + stream::iter(fspaths) + .map(move |path| { + let linter = MbtilesLinter::new(path, fix); + async move { + debug!("linting: {}", path.display()); + let start_time = std::time::Instant::now(); + + let lint_results = linter.lint().await; + let elapsed = start_time.elapsed(); + let file_results = match lint_results { + Ok(r) => FileLintResults { + fspath: path.display().to_string(), + errors: Some(r), + dt: elapsed, + }, + Err(e) => { + warn!("lint error: {}", e); + FileLintResults { + fspath: path.display().to_string(), + errors: None, + dt: elapsed, + } + } + }; + file_results + } + }) + .buffer_unordered(12) +} +pub async fn lint_filepaths( + fspaths: Vec, + fix: bool, +) -> UtilesResult> { + let mut results = lint_filepaths_stream(&fspaths, fix); + let all_lints = Vec::new(); + // let mut errors = Vec::new(); + while let Some(file_res) = results.next().await { + // let json_string = serde_json::to_string(&file_res).unwrap(); + // println!("{json_string}"); + match file_res.errors { + Some(r) => { + debug!("r: {:?}", r); + if r.is_empty() { + debug!("OK: {}", file_res.fspath); + } else { + debug!("{} - {} errors found", file_res.fspath, r.len()); + let strings = r + .iter() + .map(|e| e.format_error(&file_res.fspath.to_string())) + .collect::>(); + let joined = strings.join("\n"); + println!("{joined}"); + + for err in &r { + debug!("{}", err.to_string()); + } + } + } + None => { + debug!("OK: {}", file_res.fspath); + } + } + } + Ok(all_lints) +} diff --git a/crates/utiles/src/mbt/agg_tiles_hash.rs b/crates/utiles/src/mbt/agg_tiles_hash.rs index 60d75ffa..60d985fa 100644 --- a/crates/utiles/src/mbt/agg_tiles_hash.rs +++ b/crates/utiles/src/mbt/agg_tiles_hash.rs @@ -1,7 +1,15 @@ +use crate::errors::UtilesResult; use crate::mbt::hash_types::HashType; -use crate::sqlite::RusqliteResult; +use crate::mbt::TilesFilter; use rusqlite::Connection; - +use serde::Serialize; +#[derive(Debug, Serialize)] +pub struct AggHashResult { + pub hash_type: HashType, + pub hash: String, + pub ntiles: usize, + pub dt: std::time::Duration, +} // ================================================================= // HASH FUNCTIONS ~ HASH FUNCTIONS ~ HASH FUNCTIONS ~ HASH FUNCTIONS // ================================================================= @@ -19,50 +27,55 @@ use rusqlite::Connection; // ); // sql // } -pub fn mbt_agg_tile_hash_query(hash_type: HashType) -> String { +pub fn mbt_agg_tile_hash_query( + hash_type: HashType, + prefix: Option<&str>, + filter: &Option, +) -> UtilesResult { + let where_clause = if let Some(filter) = filter { + filter.mbtiles_sql_where(prefix)? + } else { + String::new() + }; let sql = format!( - "SELECT coalesce( - {hash_type}_concat_hex( - cast(zoom_level AS text), - cast(tile_column AS text), - cast(tile_row AS text), - tile_data - ORDER BY zoom_level, tile_column, tile_row), - {hash_type}_hex('')) - FROM tiles" + " +SELECT + coalesce( + {hash_type}_concat_hex( + cast(zoom_level AS text), + cast(tile_column AS text), + cast(tile_row AS text), + tile_data + ORDER BY zoom_level, tile_column, tile_row + ), + {hash_type}_hex('') + ) AS concatenated_hash, + COUNT(*) AS total_count +FROM tiles +{where_clause} +LIMIT 1 + ", ); - sql + Ok(sql) } pub fn mbt_agg_tiles_hash( conn: &Connection, hash_type: HashType, -) -> RusqliteResult { - let mut stmt = conn.prepare_cached(mbt_agg_tile_hash_query(hash_type).as_str())?; - let agg_tiles_hash_str: String = stmt.query_row([], |row| row.get(0))?; - Ok(agg_tiles_hash_str) + prefix: Option<&str>, + filter: &Option, +) -> UtilesResult { + let sql = mbt_agg_tile_hash_query(hash_type, prefix, filter)?; + let mut stmt = conn.prepare_cached(&sql)?; + // start time + let ti = std::time::Instant::now(); + let (agg_tiles_hash_str, count): (String, i64) = + stmt.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?; + let dt = ti.elapsed(); + Ok(AggHashResult { + hash_type, + hash: agg_tiles_hash_str, + ntiles: count as usize, + dt, + }) } -// -// pub fn mbt_agg_tiles_hash_md5(conn: &Connection) -> RusqliteResult { -// mbt_agg_tiles_hash(conn, HashType::Md5) -// } -// -// pub fn mbt_agg_tiles_hash_fnv1a(conn: &Connection) -> RusqliteResult { -// mbt_agg_tiles_hash(conn, HashType::Fnv1a) -// } -// -// pub fn mbt_agg_tiles_hash_xxh32(conn: &Connection) -> RusqliteResult { -// mbt_agg_tiles_hash(conn, HashType::Xxh32) -// } -// -// pub fn mbt_agg_tiles_hash_xxh64(conn: &Connection) -> RusqliteResult { -// mbt_agg_tiles_hash(conn, HashType::Xxh64) -// } -// -// pub fn mbt_agg_tiles_hash_xxh3_64(conn: &Connection) -> RusqliteResult { -// mbt_agg_tiles_hash(conn, HashType::Xxh3_64) -// } -// -// pub fn mbt_agg_tiles_hash_xxh3_128(conn: &Connection) -> RusqliteResult { -// mbt_agg_tiles_hash(conn, HashType::Xxh3_128) -// } diff --git a/crates/utiles/src/mbt/hash_types.rs b/crates/utiles/src/mbt/hash_types.rs index c88c91f5..a1d131e6 100644 --- a/crates/utiles/src/mbt/hash_types.rs +++ b/crates/utiles/src/mbt/hash_types.rs @@ -1,11 +1,16 @@ use crate::UtilesError; +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; use std::str::FromStr; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "kebab-case")] +#[derive(Default)] pub enum HashType { Md5, Fnv1a, Xxh32, + #[default] Xxh64, Xxh3_64, Xxh3_128, diff --git a/crates/utiles/src/mbt/mbt_stats.rs b/crates/utiles/src/mbt/mbt_stats.rs index 59ee9535..2063e4ca 100644 --- a/crates/utiles/src/mbt/mbt_stats.rs +++ b/crates/utiles/src/mbt/mbt_stats.rs @@ -1,5 +1,12 @@ -use crate::mbt::MbtType; +use rusqlite::Connection; use serde::Serialize; +use tracing::debug; + +use crate::errors::UtilesResult; +use crate::mbt::query::query_mbtiles_type; +use crate::mbt::MbtType; +use crate::sqlite::{pragma_freelist_count, pragma_page_count, pragma_page_size}; +use crate::utilesqlite::mbtiles::{zoom_stats, zoom_stats_full}; #[derive(Debug, Serialize)] pub struct MbtilesZoomStats { @@ -24,7 +31,68 @@ pub struct MbtilesStats { pub page_count: i64, pub page_size: i64, pub freelist_count: i64, - pub minzoom: Option, - pub maxzoom: Option, + pub minzoom: Option, + pub maxzoom: Option, pub zooms: Vec, } + +pub fn query_mbt_stats( + conn: &Connection, + full: Option, +) -> UtilesResult { + let query_ti = std::time::Instant::now(); + let maybe_filepath = conn.path().map(|p| p.to_string()); + let filesize = match maybe_filepath { + Some(fp) => std::fs::metadata(fp).map(|md| md.len()).unwrap_or(0), + None => 0, + }; + + // let zoom_stats_full = full.unwrap_or(false) || filesize < 10_000_000_000; + debug!("Started zoom_stats query"); + let page_count = pragma_page_count(conn)?; + + let page_size = pragma_page_size(conn, None)?; + let freelist_count = pragma_freelist_count(conn)?; + // if the file is over 10gb and full is None or false just don't do the + // zoom_stats query that counts size... bc it is slow af + // let zoom_stats = self.zoom_stats(zoom_stats_full)?; + let zoom_stats = + if full.unwrap_or(false) || (filesize < 10_000_000_000 && filesize > 0) { + zoom_stats_full(conn)? + } else { + zoom_stats(conn)? + }; + debug!("zoom_stats: {:?}", zoom_stats); + let query_dt = query_ti.elapsed(); + debug!("Finished zoom_stats query in {:?}", query_dt); + let mbt_type = query_mbtiles_type(conn)?; + if zoom_stats.is_empty() { + return Ok(MbtilesStats { + filesize, + mbtype: mbt_type, + page_count, + page_size, + freelist_count, + ntiles: 0, + minzoom: None, + maxzoom: None, + nzooms: 0, + zooms: vec![], + }); + } + + let minzoom = zoom_stats.iter().map(|r| r.zoom).min(); + let maxzoom = zoom_stats.iter().map(|r| r.zoom).max(); + Ok(MbtilesStats { + ntiles: zoom_stats.iter().map(|r| r.ntiles).sum(), + filesize, + mbtype: mbt_type, + page_count, + page_size, + freelist_count, + minzoom, + maxzoom, + nzooms: zoom_stats.len() as u32, + zooms: zoom_stats, + }) +} diff --git a/crates/utiles/src/mbt/metadata/change.rs b/crates/utiles/src/mbt/metadata/change.rs index eb96d7a1..e1bb204b 100644 --- a/crates/utiles/src/mbt/metadata/change.rs +++ b/crates/utiles/src/mbt/metadata/change.rs @@ -1,31 +1,63 @@ use crate::utilesqlite::mbtiles::{metadata_delete, metadata_set}; use json_patch::Patch; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; use tracing::warn; -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct MetadataChangeFromTo { + pub name: String, + pub from: Option, + pub to: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct MetadataChange { - pub timestamp: String, + pub changes: Vec, pub forward: Patch, pub reverse: Patch, pub data: Value, } +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum DbChangeType { + Metadata(MetadataChange), + Unknown(Value), +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct DbChangeset { + pub timestamp: String, + pub changes: Vec, +} + +impl From for DbChangeType { + fn from(change: MetadataChange) -> Self { + Self::Metadata(change) + } +} + +impl From for DbChangeset { + fn from(change: DbChangeType) -> Self { + Self { + timestamp: chrono::Utc::now().to_rfc3339(), + changes: vec![change], + } + } +} + impl MetadataChange { #[must_use] - pub fn from_forward_reverse_data( - forward: Patch, - reverse: Patch, - data: Value, - ) -> Self { - MetadataChange { - timestamp: chrono::Utc::now().to_rfc3339(), - forward, - reverse, - data, + pub fn new_empty() -> Self { + Self { + changes: vec![], + forward: Patch(vec![]), + reverse: Patch(vec![]), + data: Value::Null, } } + #[must_use] pub fn forward_keys(&self) -> Vec { self.forward diff --git a/crates/utiles/src/mbt/metadata/mod.rs b/crates/utiles/src/mbt/metadata/mod.rs index a078ab3f..c7e1861d 100644 --- a/crates/utiles/src/mbt/metadata/mod.rs +++ b/crates/utiles/src/mbt/metadata/mod.rs @@ -1,7 +1,6 @@ mod change; mod parse; mod read_fspath; - -pub use change::MetadataChange; +pub use change::{DbChangeType, DbChangeset, MetadataChange, MetadataChangeFromTo}; pub use parse::{parse_metadata_json, parse_metadata_json_value}; pub use read_fspath::read_metadata_json; diff --git a/crates/utiles/src/mbt/metadata/read_fspath.rs b/crates/utiles/src/mbt/metadata/read_fspath.rs index ff8ab74c..7159ebcb 100644 --- a/crates/utiles/src/mbt/metadata/read_fspath.rs +++ b/crates/utiles/src/mbt/metadata/read_fspath.rs @@ -2,8 +2,8 @@ use std::path::Path; use tokio::fs::read_to_string; +use crate::errors::UtilesResult; use crate::mbt::metadata_row::MbtilesMetadataJson; -use crate::UtilesResult; // read metadata json from filepath... pub async fn read_metadata_json( diff --git a/crates/utiles/src/mbt/metadata_row.rs b/crates/utiles/src/mbt/metadata_row.rs index 3f45a2da..f1eddefb 100644 --- a/crates/utiles/src/mbt/metadata_row.rs +++ b/crates/utiles/src/mbt/metadata_row.rs @@ -1,8 +1,12 @@ -use crate::UtilesResult; -use json_patch::Patch; +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::BTreeMap; +use tracing::debug; + +use crate::errors::UtilesResult; +use crate::mbt::{MetadataChange, MetadataChangeFromTo}; +use crate::UtilesError; /// Metadata row struct for `mbtiles` metadata table #[derive(Debug, Clone, Serialize, Deserialize)] @@ -103,6 +107,39 @@ impl MbtilesMetadataJson { } } + pub fn update(&mut self, key: &str, value: &str) { + let val = match value.parse::() { + Ok(v) => v, + Err(_) => Value::String(value.to_string()), + }; + match self { + MbtilesMetadataJson::Obj(obj) => { + // if key exists, and value is different update value + if let Some(v) = obj.get_mut(key) { + if v != &val { + *v = val; + } + } else { + // else insert new key value pair + obj.insert(key.to_string(), val); + } // baboom + } + MbtilesMetadataJson::Arr(arr) => { + // if key exists update value + if let Some(row) = arr.iter_mut().find(|row| row.name == key) { + if row.value != val { + row.value = val; + } + } else { + // else insert new row + arr.push(MbtilesMetadataRowParsed { + name: key.to_string(), + value: val, + }); + } + } + } + } #[must_use] pub fn as_obj(&self) -> BTreeMap { match self { @@ -117,6 +154,39 @@ impl MbtilesMetadataJson { } } } + /// Returns the raw object representation of the metadata + /// where the value is json-stringified if it is not a string + #[must_use] + pub fn as_obj_raw(&self) -> BTreeMap { + match self { + MbtilesMetadataJson::Obj(obj) => { + let obj: BTreeMap = obj + .iter() + .map(|(k, v)| { + let val = match v { + Value::String(s) => s.clone(), + _ => serde_json::to_string(v).unwrap_or_default(), + }; + (k.clone(), val) + }) + .collect(); + obj + } + MbtilesMetadataJson::Arr(arr) => { + let obj: BTreeMap = arr + .iter() + .map(|row| { + let val = match &row.value { + Value::String(s) => s.clone(), + _ => serde_json::to_string(&row.value).unwrap_or_default(), + }; + (row.name.clone(), val) + }) + .collect(); + obj + } + } + } #[must_use] pub fn as_arr(&self) -> Vec { @@ -154,22 +224,76 @@ impl MbtilesMetadataJson { } } + pub fn diff_changes( + &self, + other: &Value, + ) -> UtilesResult> { + let from_map = self.as_obj_raw(); + let to_map: BTreeMap = match other { + Value::Object(obj) => obj + .iter() + .map(|(k, v)| { + let val = match v { + Value::String(s) => s.clone(), + _ => serde_json::to_string(v).unwrap_or_default(), + }; + (k.clone(), val) + }) + .collect(), + _ => { + return Err(UtilesError::MetadataError( + "Value is not an object".to_string(), + )) + } + }; + let all_keys = from_map.keys().chain(to_map.keys()); + let changes = all_keys + .filter_map(|k| { + let from = from_map.get(k); + let to = to_map.get(k); + if from == to { + None + } else { + Some(MetadataChangeFromTo { + name: k.clone(), + from: from.cloned(), + to: to.cloned(), + }) + } + }) + .collect(); + Ok(changes) + } + pub fn diff( &self, other: &MbtilesMetadataJson, merge: bool, - ) -> UtilesResult<(Patch, Patch, Value)> { + ) -> UtilesResult { let self_value = serde_json::to_value(self)?; let mut merged = self_value.clone(); let other_value = serde_json::to_value(other)?; + + let self_value_json_string = serde_json::to_string_pretty(&self_value)?; + let other_value_json_string = serde_json::to_string_pretty(&other_value)?; + debug!("self_value: {}", self_value_json_string); + debug!("other_value: {}", other_value_json_string); if merge { + debug!("merging..."); json_patch::merge(&mut merged, &other_value); } let forward_patch = json_patch::diff(&self_value, &merged); let reverse_patch = json_patch::diff(&merged, &self_value); let mut patched_data = self_value.clone(); json_patch::patch(&mut patched_data, &forward_patch)?; - Ok((forward_patch, reverse_patch, patched_data)) + let changes = self.diff_changes(&patched_data)?; + debug!("calculated changes: {:?}", changes); + Ok(MetadataChange { + forward: forward_patch, + reverse: reverse_patch, + data: patched_data, + changes, + }) } } diff --git a/crates/utiles/src/mbt/mod.rs b/crates/utiles/src/mbt/mod.rs index 7af08227..9ac4d3f4 100644 --- a/crates/utiles/src/mbt/mod.rs +++ b/crates/utiles/src/mbt/mod.rs @@ -1,7 +1,7 @@ pub use metadata::*; pub use tiles_row::MbtTileRow; -pub use crate::mbt::mbt_stats::{MbtilesStats, MbtilesZoomStats}; +pub use crate::mbt::mbt_stats::{query_mbt_stats, MbtilesStats, MbtilesZoomStats}; pub use crate::mbt::metadata2map::{ metadata2duplicates, metadata2map, metadata2map_val, metadata_vec_has_duplicates, }; @@ -10,6 +10,7 @@ pub use crate::mbt::metadata_row::{ MbtilesMetadataRowParsed, MbtilesMetadataRows, }; pub use crate::mbt::minzoom_maxzoom::MinZoomMaxZoom; +pub use crate::mbt::tiles_filter::TilesFilter; pub use agg_tiles_hash::mbt_agg_tiles_hash; pub use mbtype::MbtType; @@ -22,5 +23,6 @@ mod metadata2map; mod metadata_row; mod minzoom_maxzoom; pub mod query; +mod tiles_filter; mod tiles_row; pub mod zxyify; diff --git a/crates/utiles/src/mbt/query.rs b/crates/utiles/src/mbt/query.rs index 796464b9..c34de708 100644 --- a/crates/utiles/src/mbt/query.rs +++ b/crates/utiles/src/mbt/query.rs @@ -1,8 +1,9 @@ +use indoc::indoc; use rusqlite::Connection; +use crate::errors::UtilesResult; use crate::mbt::MbtType; use crate::sqlite::RusqliteResult; -use crate::UtilesResult; const IS_FLAT_MBTILES_QUERY: &str = include_str!("sql/is-flat-mbtiles-query.sql"); const IS_NORM_MBTILES_QUERY: &str = include_str!("sql/is-norm-mbtiles-query.sql"); @@ -13,6 +14,9 @@ const IS_TIPPECANOE_MBTILES_QUERY: &str = const IS_PLANETILER_MBTILES_QUERY: &str = include_str!("sql/is-planetiler-mbtiles-query.sql"); +const METADATA_DUPLICATES_JSON_QUERY: &str = + include_str!("sql/mbt-metadata-duplicates-json.sql"); + pub fn is_tiles_with_hash(conn: &Connection) -> RusqliteResult { let mut stmt = conn.prepare(IS_HASH_MBTILES_QUERY)?; let r = stmt.query_row([], |row| { @@ -97,33 +101,39 @@ pub fn default_mbtiles_settings(conn: &Connection) -> UtilesResult<()> { } pub fn create_metadata_table_pk(conn: &Connection) -> RusqliteResult<()> { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS metadata ( - name TEXT PRIMARY KEY NOT NULL, - value TEXT - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE metadata ( + name TEXT PRIMARY KEY NOT NULL, + value TEXT + ) + " + })?; Ok(()) } pub fn create_metadata_table_nopk(conn: &Connection) -> RusqliteResult<()> { - conn.execute_batch( - "CREATE TABLE metadata ( - name TEXT NOT NULL, - value TEXT - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE metadata ( + name TEXT NOT NULL, + value TEXT + ) + " + })?; Ok(()) } pub fn create_metadata_table_if_not_exists_nopk( conn: &Connection, ) -> RusqliteResult<()> { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS metadata ( - name TEXT NOT NULL, - value TEXT - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE IF NOT EXISTS metadata ( + name TEXT NOT NULL, + value TEXT + ) + " + })?; Ok(()) } @@ -144,25 +154,29 @@ pub fn create_tiles_table_flat_pk( if_not_exists: bool, ) -> RusqliteResult<()> { if if_not_exists { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS tiles ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_data BLOB, - PRIMARY KEY (zoom_level, tile_column, tile_row) - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE IF NOT EXISTS tiles ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_data BLOB, + PRIMARY KEY (zoom_level, tile_column, tile_row) + ) + " + })?; } else { - conn.execute_batch( - "CREATE TABLE tiles ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_data BLOB, - PRIMARY KEY (zoom_level, tile_column, tile_row) - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE tiles ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_data BLOB, + PRIMARY KEY (zoom_level, tile_column, tile_row) + ) + " + })?; } Ok(()) } @@ -172,23 +186,27 @@ pub fn create_tiles_table_flat( if_not_exists: bool, ) -> RusqliteResult<()> { if if_not_exists { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS tiles ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_data BLOB - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE IF NOT EXISTS tiles ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_data BLOB + ) + " + })?; } else { - conn.execute_batch( - "CREATE TABLE tiles ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_data BLOB - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE tiles ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_data BLOB + ) + " + })?; } Ok(()) } @@ -198,25 +216,29 @@ pub fn create_tiles_table_hash( if_not_exists: bool, ) -> RusqliteResult<()> { if if_not_exists { - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS tiles_with_hash ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_data BLOB, - tile_hash TEXT, - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE IF NOT EXISTS tiles_with_hash ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_data BLOB, + tile_hash TEXT + ) + " + })?; } else { - conn.execute_batch( - "CREATE TABLE tiles_with_hash ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_data BLOB, - tile_hash TEXT, - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE tiles_with_hash ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_data BLOB, + tile_hash TEXT + ) + " + })?; } Ok(()) } @@ -239,20 +261,24 @@ pub fn create_tiles_view_hash(conn: &Connection) -> RusqliteResult<()> { } pub fn create_mbtiles_tables_norm(conn: &Connection) -> RusqliteResult<()> { - conn.execute_batch( - "CREATE TABLE map ( - zoom_level INTEGER NOT NULL, - tile_column INTEGER NOT NULL, - tile_row INTEGER NOT NULL, - tile_id TEXT - )", - )?; - conn.execute_batch( - "CREATE TABLE images ( - tile_id TEXT NOT NULL, - tile_data BLOB NOT NULL - )", - )?; + conn.execute_batch(indoc! { + " + CREATE TABLE map ( + zoom_level INTEGER NOT NULL, + tile_column INTEGER NOT NULL, + tile_row INTEGER NOT NULL, + tile_id TEXT + ) + " + })?; + conn.execute_batch(indoc! { + " + CREATE TABLE images ( + tile_id TEXT NOT NULL, + tile_data BLOB NOT NULL + ) + " + })?; Ok(()) } @@ -265,16 +291,27 @@ pub fn create_mbtiles_indexes_norm(conn: &Connection) -> RusqliteResult<()> { } pub fn create_mbtiles_tiles_view_norm(conn: &Connection) -> RusqliteResult<()> { - conn.execute_batch( - "CREATE VIEW tiles AS - SELECT - map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data - FROM - map JOIN images - ON images.tile_id = map.tile_id;", - )?; + conn.execute_batch(indoc! { + " + CREATE VIEW tiles AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM + map JOIN images + ON images.tile_id = map.tile_id; + " + })?; Ok(()) } + +pub fn metadata_duplicates_json_query(conn: &Connection) -> RusqliteResult { + let mut stmt = conn.prepare(METADATA_DUPLICATES_JSON_QUERY)?; + let r = stmt.query_row([], |row| { + let a: String = row.get(0)?; + Ok(a) + })?; + Ok(r) +} diff --git a/crates/utiles/src/mbt/sql/mbt-metadata-duplicates-json.sql b/crates/utiles/src/mbt/sql/mbt-metadata-duplicates-json.sql new file mode 100644 index 00000000..669912e0 --- /dev/null +++ b/crates/utiles/src/mbt/sql/mbt-metadata-duplicates-json.sql @@ -0,0 +1,35 @@ +WITH + duplicate_vals AS ( + SELECT + name, + json_group_array(value) AS vals + FROM + ( + SELECT + name, + value + FROM + metadata + WHERE + name IN ( + SELECT + name + FROM + metadata + GROUP BY + name + HAVING + count(*) > 1 + ) + ORDER BY + name + ) + GROUP BY + name + ORDER BY + name + ) +SELECT + json_group_object(name, json(vals)) AS metadata +FROM + duplicate_vals; diff --git a/crates/utiles/src/mbt/tiles_filter.rs b/crates/utiles/src/mbt/tiles_filter.rs new file mode 100644 index 00000000..c31980f0 --- /dev/null +++ b/crates/utiles/src/mbt/tiles_filter.rs @@ -0,0 +1,73 @@ +use utiles_core::{tile_ranges, BBox, ZoomOrZooms, ZoomSet}; + +use crate::errors::UtilesResult; + +#[derive(Debug)] +pub struct TilesFilter { + pub bboxes: Option>, + pub zooms: Option>, +} + +impl TilesFilter { + #[must_use] + pub fn new(bboxes: Option>, zooms: Option>) -> Self { + Self { bboxes, zooms } + } + + pub fn mbtiles_sql_where(&self, prefix: Option<&str>) -> UtilesResult { + self.where_clause(prefix) + } + + pub fn where_clause(&self, prefix: Option<&str>) -> UtilesResult { + let pred = match (&self.bboxes, &self.zooms) { + (Some(bbox), Some(zooms)) => { + let zboxes = bbox + .iter() + .flat_map(|b| { + tile_ranges(b.tuple(), ZoomOrZooms::Zooms(zooms.clone())) + }) + .collect::>(); + let pred = zboxes + .iter() + .map(|a| a.mbtiles_sql_where(prefix)) + .collect::>() + .join(" OR "); + format!("({pred})") + } + (Some(bbox), None) => { + let zboxes = bbox + .iter() + .flat_map(|b| { + tile_ranges( + b.tuple(), + ZoomOrZooms::Zooms(ZoomSet::all().into()), + ) + }) + .collect::>(); + let pred = zboxes + .iter() + .map(|a| a.mbtiles_sql_where(prefix)) + .collect::>() + .join(" OR "); + format!("({pred})") + } + (None, Some(zooms)) => { + format!( + "zoom_level IN ({zooms})", + zooms = zooms + .iter() + .map(std::string::ToString::to_string) + .collect::>() + .join(",") + ) + } + (None, None) => String::new(), + }; + // attach 'WHERE' + if pred.is_empty() { + Ok(pred) + } else { + Ok(format!("WHERE {pred}")) + } + } +} diff --git a/crates/utiles/src/server/mod.rs b/crates/utiles/src/server/mod.rs index 57af336b..0f31d4a0 100644 --- a/crates/utiles/src/server/mod.rs +++ b/crates/utiles/src/server/mod.rs @@ -1,5 +1,4 @@ #![allow(clippy::unwrap_used)] - use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; @@ -21,6 +20,7 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use tilejson::TileJSON; use tower::ServiceBuilder; +use tower_http::compression::CompressionLayer; use tower_http::request_id::{PropagateRequestIdLayer, SetRequestIdLayer}; use tower_http::trace::{DefaultOnBodyChunk, DefaultOnFailure, DefaultOnRequest}; use tower_http::{ @@ -29,14 +29,15 @@ use tower_http::{ }; use tracing::{debug, info, warn}; +use request_id::Radix36MakeRequestId; +use utiles_core::tile_type::blob2headers; +use utiles_core::{quadkey2tile, utile, Tile}; + use crate::errors::UtilesResult; use crate::globster::find_filepaths; use crate::signal::shutdown_signal; use crate::utilesqlite::mbtiles_async::MbtilesAsync; use crate::utilesqlite::MbtilesAsyncSqliteClient; -use request_id::Radix36MakeRequestId; -use utiles_core::tile_type::blob2headers; -use utiles_core::{quadkey2tile, utile, Tile}; pub mod radix36; mod request_id; @@ -123,12 +124,9 @@ async fn preflight(config: &UtilesServerConfig) -> UtilesResult { Ok(Datasets { mbtiles: datasets }) } -pub async fn utiles_serve( - cfg: UtilesServerConfig, -) -> Result<(), Box> { +pub async fn utiles_serve(cfg: UtilesServerConfig) -> UtilesResult<()> { info!("__UTILES_SERVE__"); - let utiles_serve_config_json = serde_json::to_string_pretty(&cfg) - .expect("Failed to serialize utiles_serve_config_json"); + let utiles_serve_config_json = serde_json::to_string_pretty(&cfg)?; info!("config:\n{}", utiles_serve_config_json); let addr = cfg.addr(); @@ -143,7 +141,12 @@ pub async fn utiles_serve( // ...seems to be the idiomatic way to do this... let shared_state = Arc::new(state); let x_request_id = HeaderName::from_static("x-request-id"); - + let comression_layer: CompressionLayer = CompressionLayer::new() + // .br(true) + // .deflate(true) + .gzip(true) + .zstd(true); + // .compress_when(|_, _, _, _| true); // Build our middleware stack let middleware = ServiceBuilder::new() .layer(SetRequestIdLayer::new( @@ -161,7 +164,8 @@ pub async fn utiles_serve( ) // propagate `x-request-id` headers from request to response .layer(PropagateRequestIdLayer::new(x_request_id)) - .layer(TimeoutLayer::new(Duration::from_secs(10))); + .layer(TimeoutLayer::new(Duration::from_secs(10))) + .layer(comression_layer); // Build the app/router! let app = Router::new() @@ -178,9 +182,7 @@ pub async fn utiles_serve( // let addr = cfg.addr(); info!("Listening on: {}", addr); - let listener = tokio::net::TcpListener::bind(addr) - .await - .expect("Failed to bind to address"); + let listener = tokio::net::TcpListener::bind(addr).await?; axum::serve(listener, app) .with_graceful_shutdown(shutdown_signal()) .await?; diff --git a/crates/utiles/src/sqlite/async_sqlite3.rs b/crates/utiles/src/sqlite/async_sqlite3.rs index 1ef1f90e..37d35de4 100644 --- a/crates/utiles/src/sqlite/async_sqlite3.rs +++ b/crates/utiles/src/sqlite/async_sqlite3.rs @@ -1,21 +1,98 @@ -use std::fmt::Debug; - -use async_sqlite::{Client, Error as AsyncSqliteError}; -use async_trait::async_trait; -use rusqlite::Connection; - +use crate::fs_async::file_exists; use crate::sqlite::sqlike3::Sqlike3Async; use crate::sqlite::{ analyze, attach_db, detach_db, is_empty_db, pragma_freelist_count, pragma_index_list, pragma_page_count, pragma_page_size, pragma_page_size_set, - pragma_table_list, vacuum, vacuum_into, PragmaIndexListRow, PragmaTableListRow, - SqliteResult, + pragma_table_list, vacuum, vacuum_into, DbPath, PragmaIndexListRow, + PragmaTableListRow, SqliteError, SqliteResult, }; +use async_sqlite::{Client, ClientBuilder, Error as AsyncSqliteError}; +use async_trait::async_trait; +use rusqlite::{Connection, OpenFlags}; +use std::fmt; +use std::fmt::Debug; +use std::path::Path; +use tracing::debug; pub struct SqliteDbAsyncClient { + pub dbpath: DbPath, pub client: Client, } +#[allow(clippy::missing_fields_in_debug)] +impl Debug for SqliteDbAsyncClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // use the dbpath to debug + f.debug_struct("SqliteDbAsyncClient") + .field("fspath", &self.dbpath.fspath) + .finish() + } +} + +impl SqliteDbAsyncClient { + pub async fn new(client: Client, dbpath: Option) -> SqliteResult { + if let Some(dbpath) = dbpath { + Ok(Self { dbpath, client }) + } else { + let path = client + .conn(|conn| { + let maybe_path = conn.path(); + + let path = maybe_path.unwrap_or("unknown").to_string(); + Ok(path) + }) + .await?; + Ok(Self { + client, + dbpath: DbPath::new(&path), + }) + } + } + + pub async fn open>( + path: P, + open_flags: Option, + ) -> SqliteResult { + debug!("Opening sqlite db with client: {}", path.as_ref().display()); + let client = ClientBuilder::new() + .path(&path) + .flags(open_flags.unwrap_or_default()) + .open() + .await?; + Ok({ + Self { + dbpath: DbPath::new(path.as_ref().to_str().unwrap_or_default()), + client, + } + }) + } + + pub async fn open_readonly>(path: P) -> SqliteResult { + let flags = OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_URI; + debug!( + "Opening sqlite db readonly with client: {}", + path.as_ref().display() + ); + SqliteDbAsyncClient::open(path, Some(flags)).await + } + pub async fn open_existing>( + path: P, + flags: Option, + ) -> SqliteResult { + debug!( + "Opening sqlite db existing with client: {}", + path.as_ref().display() + ); + if !file_exists(&path).await { + return Err(SqliteError::FileDoesNotExist( + path.as_ref().display().to_string(), + )); + } + SqliteDbAsyncClient::open(path, flags).await + } +} #[async_trait] pub trait AsyncSqliteConn: Send + Sync { async fn conn(&self, func: F) -> Result @@ -24,6 +101,14 @@ pub trait AsyncSqliteConn: Send + Sync { T: Send + 'static; } +#[async_trait] +pub trait AsyncSqliteConnMut: Send + Sync { + async fn conn_mut(&self, func: F) -> Result + where + F: FnOnce(&mut Connection) -> Result + Send + 'static, + T: Send + 'static; +} + #[async_trait] impl AsyncSqliteConn for SqliteDbAsyncClient { async fn conn(&self, func: F) -> Result @@ -35,6 +120,17 @@ impl AsyncSqliteConn for SqliteDbAsyncClient { } } +#[async_trait] +impl AsyncSqliteConnMut for SqliteDbAsyncClient { + async fn conn_mut(&self, func: F) -> Result + where + F: FnOnce(&mut Connection) -> Result + Send + 'static, + T: Send + 'static, + { + self.client.conn_mut(func).await + } +} + #[async_trait] impl Sqlike3Async for T where diff --git a/crates/utiles/src/sqlite/db.rs b/crates/utiles/src/sqlite/db.rs index 45c9430f..e07e2bbf 100644 --- a/crates/utiles/src/sqlite/db.rs +++ b/crates/utiles/src/sqlite/db.rs @@ -1,14 +1,14 @@ use std::collections::HashMap; use std::path::Path; +use rusqlite::Connection; + use crate::errors::UtilesResult; use crate::sqlite::{ pragma_database_list, pragma_index_list, PragmaIndexListRow, RusqliteResult, Sqlike3, }; use crate::UtilesError; -use rusqlite::Connection; -use tracing::debug; pub struct SqliteDb { pub conn: Connection, @@ -18,6 +18,10 @@ impl Sqlike3 for SqliteDb { fn conn(&self) -> &Connection { &self.conn } + + fn conn_mut(&mut self) -> &mut Connection { + &mut self.conn + } } impl SqliteDb { @@ -72,48 +76,6 @@ pub fn pragma_index_list_all_tables( } Ok(index_map) } - -pub fn application_id(conn: &Connection) -> RusqliteResult { - let mut stmt = conn.prepare("PRAGMA application_id")?; - let mut rows = stmt.query([])?; - let row = rows - .next()? - .expect("'PRAGMA application_id' -- should return row but did not"); - let app_id: u32 = row.get(0)?; - Ok(app_id) -} - -pub fn application_id_set(conn: &Connection, app_id: u32) -> RusqliteResult { - let current_app_id = application_id(conn)?; - if current_app_id == app_id { - debug!("application_id_set: current app_id == app_id: {}", app_id); - Ok(current_app_id) - } else { - debug!( - "application_id_set: current app_id != app_id: {} != {}", - current_app_id, app_id - ); - let sql = format!("PRAGMA application_id = {app_id}"); - let mut stmt = conn.prepare(&sql)?; - stmt.execute([])?; - Ok(app_id) - } -} - -pub fn journal_mode(conn: &Connection) -> RusqliteResult { - let mut stmt = conn.prepare("PRAGMA journal_mode")?; - let mut rows = stmt.query([])?; - let row = rows - .next()? - .expect("'PRAGMA journal_mode' -- should return row but did not"); - let jm: String = row.get(0)?; - Ok(jm) -} - -pub fn magic_number(conn: &Connection) -> RusqliteResult { - application_id(conn) -} - pub fn query_db_fspath(conn: &Connection) -> RusqliteResult> { let rows = pragma_database_list(conn)?; let row = rows.iter().find_map(|r| { @@ -125,7 +87,6 @@ pub fn query_db_fspath(conn: &Connection) -> RusqliteResult> { }); Ok(row) } - pub fn is_empty_db(connection: &Connection) -> RusqliteResult { let mut stmt = connection.prepare("SELECT COUNT(*) FROM sqlite_schema")?; let rows = stmt.query_row([], |row| { diff --git a/crates/utiles/src/utilesqlite/dbpath.rs b/crates/utiles/src/sqlite/dbpath.rs similarity index 80% rename from crates/utiles/src/utilesqlite/dbpath.rs rename to crates/utiles/src/sqlite/dbpath.rs index 31b33a71..f2c0a1a6 100644 --- a/crates/utiles/src/utilesqlite/dbpath.rs +++ b/crates/utiles/src/sqlite/dbpath.rs @@ -1,4 +1,5 @@ use crate::errors::UtilesResult; +use crate::fs_async::file_exists; use crate::UtilesError; use std::path::PathBuf; use tracing::debug; @@ -18,21 +19,19 @@ impl std::fmt::Display for DbPath { } impl DbPath { + #[must_use] pub fn new(fspath: &str) -> Self { - let p = PathBuf::from(fspath); - let filename = p - .file_name() - .expect("DbPath::new: invalid path, could not get filename from path"); - DbPath { - fspath: fspath.to_string(), - filename: filename - .to_str() - .expect( - "DbPath::new: invalid path, could not convert filename to string", - ) - .to_string(), - } + // let p = PathBuf::from(fspath); + pathlike2dbpath(fspath).map_or( + DbPath { + fspath: "unknown".to_string(), + filename: "unknown".to_string(), + }, + |a| a, + ) } + + #[must_use] pub fn memory() -> Self { DbPath { fspath: ":memory:".to_string(), @@ -40,9 +39,15 @@ impl DbPath { } } + #[must_use] pub fn fspath_exists(&self) -> bool { PathBuf::from(&self.fspath).exists() } + + #[must_use] + pub async fn fspath_exists_async(&self) -> bool { + file_exists(&self.fspath).await + } } // try from for dbpath diff --git a/crates/utiles/src/sqlite/errors.rs b/crates/utiles/src/sqlite/errors.rs index c6482d20..f82fd4a2 100644 --- a/crates/utiles/src/sqlite/errors.rs +++ b/crates/utiles/src/sqlite/errors.rs @@ -15,6 +15,9 @@ pub enum SqliteError { #[allow(clippy::enum_variant_names)] #[error("sqlite err: {0}")] AsyncSqliteError(async_sqlite::Error), + + #[error("File does not exist: {0}")] + FileDoesNotExist(String), } pub type SqliteResult = Result; diff --git a/crates/utiles/src/sqlite/insert_strategy.rs b/crates/utiles/src/sqlite/insert_strategy.rs index 78880166..4d974b3e 100644 --- a/crates/utiles/src/sqlite/insert_strategy.rs +++ b/crates/utiles/src/sqlite/insert_strategy.rs @@ -1,6 +1,8 @@ use serde::Serialize; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize)] +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, strum_macros::Display, +)] pub enum InsertStrategy { #[default] None, diff --git a/crates/utiles/src/sqlite/mod.rs b/crates/utiles/src/sqlite/mod.rs index 1b52840d..bc575167 100644 --- a/crates/utiles/src/sqlite/mod.rs +++ b/crates/utiles/src/sqlite/mod.rs @@ -4,12 +4,13 @@ pub use rusqlite::{Connection, Result as RusqliteResult}; pub use affected::{AffectedType, RowsAffected}; -pub use async_sqlite3::{AsyncSqliteConn, SqliteDbAsyncClient}; +pub use async_sqlite3::{AsyncSqliteConn, AsyncSqliteConnMut, SqliteDbAsyncClient}; pub use attach::{attach_db, detach_db}; pub use db::*; +pub use dbpath::*; pub use errors::{SqliteError, SqliteResult}; pub use insert_strategy::InsertStrategy; -pub use page_size::{is_valid_page_size, pragma_page_size_get}; +pub use page_size::is_valid_page_size; pub use pragma::*; pub use sqlike3::{Sqlike3, Sqlike3Async}; @@ -17,6 +18,7 @@ mod affected; mod async_sqlite3; mod attach; mod db; +mod dbpath; mod errors; mod insert_strategy; mod page_size; diff --git a/crates/utiles/src/sqlite/page_size.rs b/crates/utiles/src/sqlite/page_size.rs index 05875012..3c09cbcf 100644 --- a/crates/utiles/src/sqlite/page_size.rs +++ b/crates/utiles/src/sqlite/page_size.rs @@ -1,9 +1,5 @@ //! Page size tools for sqlite -use rusqlite::Connection; - -use crate::sqlite::RusqliteResult; - /// Return true if `page_size` is valid; power of 2 between 512 and 65536. /// /// Ref: [SQLite Page Size](https://www.sqlite.org/pragma.html#pragma_page_size) @@ -19,18 +15,3 @@ pub fn is_valid_page_size(page_size: i64) -> bool { || page_size == 32768 || page_size == 65536 } - -pub fn pragma_page_size_get(conn: &Connection) -> RusqliteResult { - let mut stmt = conn.prepare("PRAGMA page_size")?; - let mut rows = stmt.query([])?; - let row = rows - .next() - .expect("PRAGMA page_size -- should return but did not"); - match row { - Some(row) => { - let page_size: i64 = row.get(0)?; - Ok(page_size) - } - None => Err(rusqlite::Error::QueryReturnedNoRows), - } -} diff --git a/crates/utiles/src/sqlite/pragma.rs b/crates/utiles/src/sqlite/pragma.rs index fc1944b4..01233ac9 100644 --- a/crates/utiles/src/sqlite/pragma.rs +++ b/crates/utiles/src/sqlite/pragma.rs @@ -1,7 +1,35 @@ -use rusqlite::{Connection, Error as RusqliteError, Result as RusqliteResult}; +use rusqlite::{ + Connection, DatabaseName, Error as RusqliteError, Result as RusqliteResult, +}; +use tracing::debug; -use crate::sqlite::page_size::pragma_page_size_get; +pub fn journal_mode(conn: &Connection) -> RusqliteResult { + let jm = conn.pragma_query_value(None, "journal_mode", |row| row.get(0))?; + Ok(jm) +} + +pub fn journal_mode_set( + conn: &Connection, + mode: &str, + schema_name: Option, +) -> RusqliteResult { + let current_mode = conn.pragma_query_value(schema_name, "journal_mode", |row| { + let val: String = row.get(0)?; + Ok(val) + })?; + if current_mode == mode { + debug!("journal_mode_set: current mode == mode: {}", mode); + Ok(false) + } else { + debug!( + "journal_mode_set: current mode != mode: {} != {}", + current_mode, mode + ); + conn.pragma_update(schema_name, "journal_mode", mode)?; + Ok(true) + } +} pub fn pragma_page_count(conn: &Connection) -> RusqliteResult { let mut stmt = conn.prepare("PRAGMA page_count")?; let mut rows = stmt.query([])?; @@ -10,12 +38,52 @@ pub fn pragma_page_count(conn: &Connection) -> RusqliteResult { Ok(count) } +pub fn application_id(conn: &Connection) -> RusqliteResult { + let app_id = conn.pragma_query_value(None, "application_id", |row| row.get(0))?; + Ok(app_id) +} + +pub fn application_id_set(conn: &Connection, app_id: u32) -> RusqliteResult { + let current_app_id = application_id(conn)?; + if current_app_id == app_id { + debug!("application_id_set: current app_id == app_id: {}", app_id); + Ok(current_app_id) + } else { + debug!( + "application_id_set: current app_id != app_id: {} != {}", + current_app_id, app_id + ); + conn.pragma_update(None, "application_id", app_id)?; + Ok(app_id) + } +} + +pub fn magic_number(conn: &Connection) -> RusqliteResult { + application_id(conn) +} + pub fn pragma_freelist_count(conn: &Connection) -> RusqliteResult { - let mut stmt = conn.prepare("PRAGMA freelist_count")?; - let mut rows = stmt.query([])?; - let row = rows.next()?.ok_or(RusqliteError::QueryReturnedNoRows)?; - let count: i64 = row.get(0)?; - Ok(count) + let freelist_count = conn.pragma_query_value(None, "freelist_count", |row| { + let count: i64 = row.get(0)?; + Ok(count) + })?; + Ok(freelist_count) +} + +pub fn pragma_page_size_get(conn: &Connection) -> RusqliteResult { + let r = conn.pragma_query_value(None, "page_size", |row| row.get(0))?; + Ok(r) +} + +pub fn pragma_page_size_set(conn: &Connection, page_size: i64) -> RusqliteResult { + // set page size + let current_page_size = pragma_page_size_get(conn)?; + if current_page_size == page_size { + Ok(page_size) + } else { + conn.pragma_update(None, "page_size", page_size)?; + Ok(page_size) + } } pub fn pragma_page_size( @@ -29,17 +97,6 @@ pub fn pragma_page_size( } } -pub fn pragma_page_size_set(conn: &Connection, page_size: i64) -> RusqliteResult { - // set page size - let current_page_size = pragma_page_size_get(conn)?; - if current_page_size == page_size { - return Ok(page_size); - } - let stmt_str = format!("PRAGMA page_size = {page_size}"); - conn.execute(&stmt_str, [])?; - Ok(page_size) -} - #[derive(Debug)] pub struct PragmaTableListRow { pub schema: String, @@ -224,3 +281,38 @@ pub fn pragma_index_info( let rows = mapped_rows.collect::>>()?; Ok(rows) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use crate::sqlite::open; + + use super::*; + + #[test] + fn journal_mode_pragma() { + let conn = open(":memory:").unwrap(); + let jm = journal_mode(&conn).unwrap(); + assert_eq!(jm, "memory"); + let jm_set_res = journal_mode_set(&conn, "wal", None).unwrap(); + assert!(jm_set_res); + } + + #[test] + fn pragma_page_size_pragma() { + let conn = open(":memory:").unwrap(); + let page_size = pragma_page_size(&conn, None).unwrap(); + assert_eq!(page_size, 4096); + let page_size_set_res = pragma_page_size_set(&conn, 8192).unwrap(); + assert_eq!(page_size_set_res, 8192); + } + + #[test] + fn pragma_application_id_pragma() { + let conn = open(":memory:").unwrap(); + let app_id = application_id(&conn).unwrap(); + assert_eq!(app_id, 0); + let app_id_set_res = application_id_set(&conn, 1).unwrap(); + assert_eq!(app_id_set_res, 1); + } +} diff --git a/crates/utiles/src/sqlite/sqlike3.rs b/crates/utiles/src/sqlite/sqlike3.rs index 9b2587ec..3e99fc98 100644 --- a/crates/utiles/src/sqlite/sqlike3.rs +++ b/crates/utiles/src/sqlite/sqlike3.rs @@ -1,11 +1,10 @@ use async_trait::async_trait; use crate::sqlite::errors::SqliteResult; -use crate::sqlite::page_size::pragma_page_size_get; use crate::sqlite::{ analyze, attach_db, detach_db, is_empty_db, pragma_freelist_count, - pragma_index_list, pragma_page_count, pragma_page_size_set, pragma_table_list, - vacuum, vacuum_into, PragmaIndexListRow, PragmaTableListRow, + pragma_index_list, pragma_page_count, pragma_page_size_get, pragma_page_size_set, + pragma_table_list, vacuum, vacuum_into, PragmaIndexListRow, PragmaTableListRow, }; macro_rules! sqlike3_methods { @@ -16,6 +15,7 @@ macro_rules! sqlike3_methods { ) => { pub trait $trait_name { fn conn(&self) -> &rusqlite::Connection; + fn conn_mut(&mut self) -> &mut rusqlite::Connection; $( fn $fn_name(&self, $($arg_name: $arg_type),*) -> $ret_type { diff --git a/crates/utiles/src/sqlite_utiles/hash_int.rs b/crates/utiles/src/sqlite_utiles/hash_int.rs index 1319ffce..d95ff8ac 100644 --- a/crates/utiles/src/sqlite_utiles/hash_int.rs +++ b/crates/utiles/src/sqlite_utiles/hash_int.rs @@ -1,16 +1,20 @@ +use std::hash::Hasher; + +use fnv; +use fnv::FnvHasher; use rusqlite::functions::FunctionFlags; use rusqlite::types::ValueRef; use rusqlite::Connection; use rusqlite::Error::{InvalidFunctionParameterType, InvalidParameterCount}; -use tracing::{debug, error}; +use tracing::{error, trace}; use xxhash_rust::const_xxh3::xxh3_64 as const_xxh3; use xxhash_rust::const_xxh64::xxh64 as const_xxh64; /// Return xxh3-64 hash of string/blob as an integer (i64) value. /// /// Sqlite stores integers as 8-byte signed integers. -pub fn add_function_xxh3_int(db: &Connection) -> rusqlite::Result<()> { - debug!("Adding xxh3_int function"); +pub fn add_function_xxh3_i64(db: &Connection) -> rusqlite::Result<()> { + trace!("Adding xxh3_i64 function"); db.create_scalar_function( "xxh3_i64", 1, @@ -39,8 +43,8 @@ pub fn add_function_xxh3_int(db: &Connection) -> rusqlite::Result<()> { /// Return xxh32 hash of string/blob as integer (i64) value. /// -pub fn add_function_xxh64_int(db: &Connection) -> rusqlite::Result<()> { - debug!("Adding xxh32_int function"); +pub fn add_function_xxh64_i64(db: &Connection) -> rusqlite::Result<()> { + trace!("Adding xxh64_i64 function"); db.create_scalar_function( "xxh64_i64", 1, @@ -65,3 +69,36 @@ pub fn add_function_xxh64_int(db: &Connection) -> rusqlite::Result<()> { }, ) } + +#[inline] +fn fnv1a_u64(bytes: &[u8]) -> u64 { + let mut hasher = FnvHasher::default(); + hasher.write(bytes); + hasher.finish() +} + +pub fn add_function_fnv_i64(db: &Connection) -> rusqlite::Result<()> { + trace!("Adding fnv_i64 function"); + db.create_scalar_function( + "fnv_i64", + 1, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + move |ctx| { + if ctx.len() != 1 { + error!("called with unexpected number of arguments"); + return Err(InvalidParameterCount(ctx.len(), 1)); + } + let raw = ctx.get_raw(0); + match raw { + ValueRef::Blob(b) | ValueRef::Text(b) => { + let hash_i64_be = i64::from_be_bytes(fnv1a_u64(b).to_be_bytes()); + Ok(rusqlite::types::Value::Integer(hash_i64_be)) + } + v => { + error!("called with unexpected argument type"); + Err(InvalidFunctionParameterType(0, v.data_type())) + } + } + }, + ) +} diff --git a/crates/utiles/src/sqlite_utiles/mod.rs b/crates/utiles/src/sqlite_utiles/mod.rs index 17d4f557..21ce413e 100644 --- a/crates/utiles/src/sqlite_utiles/mod.rs +++ b/crates/utiles/src/sqlite_utiles/mod.rs @@ -1,9 +1,17 @@ -use rusqlite::{Connection, Result}; -use tracing::debug; - -use crate::sqlite_utiles::hash_int::{add_function_xxh3_int, add_function_xxh64_int}; +//! utiles-sqlite ~ sqlite extension function(s) for utiles +//! +//! Adds the following functions: +//! - `ut_tiletype(blob)` - returns the tile type of the blob +//! - `ut_tilesize(blob)` - returns the size of raster tile or None +//! - `xxh3_int(blob|str)` - returns xxh3 hash as `i64` big-endian view +//! - `xxh64_int(blob|str)` - returns xxh64 hash as `i64` big-endian view +use crate::sqlite_utiles::hash_int::{ + add_function_fnv_i64, add_function_xxh3_i64, add_function_xxh64_i64, +}; use crate::sqlite_utiles::tilesize::add_function_ut_tilesize; use crate::sqlite_utiles::tiletype::add_function_ut_tiletype; +use rusqlite::{Connection, Result}; +use tracing::debug; mod hash_int; mod tilesize; @@ -14,8 +22,9 @@ pub fn add_ut_functions(db: &Connection) -> Result<()> { add_function_ut_tiletype(db)?; add_function_ut_tilesize(db)?; - add_function_xxh3_int(db)?; - add_function_xxh64_int(db)?; + add_function_xxh3_i64(db)?; + add_function_xxh64_i64(db)?; + add_function_fnv_i64(db)?; debug!("registered sqlite-utiles functions!"); Ok(()) } diff --git a/crates/utiles/src/tests/core.rs b/crates/utiles/src/tests/core.rs new file mode 100644 index 00000000..7c754be2 --- /dev/null +++ b/crates/utiles/src/tests/core.rs @@ -0,0 +1,109 @@ +use std::collections::HashSet; + +use crate::*; + +#[test] +fn zoom_or_zooms() { + let z = as_zooms(1.into()); + assert_eq!(z, vec![1]); + let z = as_zooms(vec![1, 2, 3].into()); + assert_eq!(z, vec![1, 2, 3]); +} + +#[test] +fn tiles_generator() { + let bounds = (-105.0, 39.99, -104.99, 40.0); + let tiles = tiles(bounds, vec![14].into()); + let expect = vec![Tile::new(3413, 6202, 14), Tile::new(3413, 6203, 14)]; + assert_eq!(tiles.collect::>(), expect); +} + +#[test] +fn tiles_single_zoom() { + let bounds = (-105.0, 39.99, -104.99, 40.0); + let tiles = tiles(bounds, 14.into()); + let expect = vec![Tile::new(3413, 6202, 14), Tile::new(3413, 6203, 14)]; + assert_eq!(tiles.collect::>(), expect); + + let num_tiles = tiles_count(bounds, 14.into()).unwrap(); + assert_eq!(num_tiles, 2); +} + +#[test] +fn tiles_anti_meridian() { + let bounds = (175.0, 5.0, -175.0, 10.0); + let mut tiles: Vec = tiles(bounds, 2.into()).collect(); + tiles.sort(); + let mut expected = vec![Tile::new(3, 1, 2), Tile::new(0, 1, 2)]; + expected.sort(); + assert_eq!(tiles, expected); +} + +#[test] +fn tile_is_valid() { + let valid_tiles = vec![ + Tile::new(0, 0, 0), + Tile::new(0, 0, 1), + Tile::new(1, 1, 1), + Tile::new(243, 166, 9), + ]; + + for tile in valid_tiles { + assert!(tile.valid(), "{tile:?} is not valid"); + } +} + +#[test] +fn tile_is_invalid() { + let invalid_tiles = vec![ + Tile::new(0, 1, 0), + Tile::new(1, 0, 0), + Tile::new(1, 1, 0), + Tile::new(1, 234, 1), + ]; + + for tile in invalid_tiles { + assert!(!tile.valid(), "{tile:?} is valid"); + } +} + +#[test] +fn test_macro() { + let tile = utile!(0, 0, 0); + assert_eq!(tile, Tile::new(0, 0, 0)); +} + +#[test] +fn test_simplify() { + let children = utile!(243, 166, 9).children(Some(12)); + assert_eq!(children.len(), 64); + let mut children = children.into_iter().collect::>(); + children.truncate(61); + children.push(children[0]); + let simplified = simplify(children.into_iter().collect::>()); + let targets = vec![ + utile!(487, 332, 10), + utile!(486, 332, 10), + utile!(487, 333, 10), + utile!(973, 667, 11), + utile!(973, 666, 11), + utile!(972, 666, 11), + utile!(1944, 1334, 12), + ]; + for target in targets { + assert!(simplified.contains(&target)); + } +} + +#[test] +fn test_simplify_removal() { + let tiles = vec![ + utile!(1298, 3129, 13), + utile!(649, 1564, 12), + utile!(650, 1564, 12), + ]; + let simplified = simplify(tiles.into_iter().collect::>()); + assert!(!simplified.contains(&utile!(1298, 3129, 13))); + assert!(simplified.contains(&utile!(650, 1564, 12))); + assert!(simplified.contains(&utile!(649, 1564, 12))); +} diff --git a/crates/utiles/src/tests/mod.rs b/crates/utiles/src/tests/mod.rs new file mode 100644 index 00000000..f59ccc9d --- /dev/null +++ b/crates/utiles/src/tests/mod.rs @@ -0,0 +1,2 @@ +#![allow(clippy::unwrap_used)] +mod core; diff --git a/crates/utiles/src/utilejson.rs b/crates/utiles/src/utilejson.rs index a6a5c567..231cf952 100644 --- a/crates/utiles/src/utilejson.rs +++ b/crates/utiles/src/utilejson.rs @@ -1,6 +1,8 @@ use std::fmt::Display; +use std::ops::Deref; use std::str::FromStr; +use serde::{Deserialize, Serialize}; use serde_json::{Value as JSONValue, Value}; use tilejson::{tilejson, Bounds, Center, TileJSON}; @@ -9,6 +11,68 @@ use utiles_core::geostats::TileStats; use crate::errors::UtilesResult; use crate::mbt::MbtMetadataRow; +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +pub struct TerrainRgbo { + pub r: u8, + pub g: u8, + pub b: u8, + pub o: u8, +} + +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +pub enum Terrain { + Rgbo(TerrainRgbo), + + // 'mapbox' string + #[serde(rename = "mapbox")] + Mapbox, + + // 'terrarium' string + #[serde(rename = "terrarium")] + Terrarium, +} +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +pub struct UTileJSON { + #[serde(flatten)] + pub tj: TileJSON, + + pub minzoom: u8, + pub maxzoom: u8, + + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub terrain: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub generator: Option, +} + +/// Set any missing default values per tile-json specification +impl UTileJSON { + pub fn set_missing_defaults(&mut self) { + self.tj.set_missing_defaults(); + + if self.tj.tilejson.is_empty() { + self.tj.tilejson = "3.0.0".to_string(); + } + if self.tj.tiles.is_empty() { + self.tj.tiles = vec![]; + } + if self.tj.vector_layers.is_none() { + self.tj.vector_layers = None; + } + } +} +impl Deref for UTileJSON { + type Target = TileJSON; + + fn deref(&self) -> &Self::Target { + &self.tj + } +} + /// # Panics /// /// Panics from `serde_json::to_string_pretty` or `serde_json::to_string` @@ -75,6 +139,159 @@ pub fn metadata2tilejson(metadata: Vec) -> UtilesResult>(); + for key in keys { + if let Some(value) = obj.remove(&key) { + tj.other.insert(key, value); + } + } + } } Ok(tj) } + +// crate::tilejson maro: +// ``` +// #[macro_export] +// macro_rules! tilejson { +// ( tilejson: $ver:expr, tiles: $sources:expr $(, $tag:tt : $val:expr)* $(,)? ) => { +// $crate::TileJSON { +// $( $tag: Some($val), )* +// ..$crate::TileJSON { +// tilejson: $ver, +// tiles: $sources, +// vector_layers: None, +// attribution: None, +// bounds: None, +// center: None, +// data: None, +// description: None, +// fillzoom: None, +// grids: None, +// legend: None, +// maxzoom: None, +// minzoom: None, +// name: None, +// scheme: None, +// template: None, +// version: None, +// other: Default::default(), +// } +// } +// }; +// ( tiles: $sources:expr $(, $tag:tt : $val:expr)* $(,)? ) => { +// $crate::tilejson! { +// tilejson: "3.0.0".to_string(), +// tiles: $sources, +// $( $tag: $val , )* } +// }; +// ( $tile_source:expr $(, $tag:tt : $val:expr)* $(,)? ) => { +// $crate::tilejson! { +// tiles: vec! [ $tile_source ], +// $( $tag: $val , )* } +// }; +// } +// ``` +#[macro_export] +macro_rules! utilejson { + ( tiles: $tile_source:expr, minzoom: $minzoom:expr, maxzoom: $maxzoom:expr $(, $tag:tt : $val:expr)* $(,)?) => { + $crate::utilejson::UTileJSON { + tj: tilejson::tilejson! { + tiles: $tile_source, + $( $tag : $val, )* + }, + minzoom: $minzoom, + maxzoom: $maxzoom, + terrain: None, + id: None, + generator: None, + } + }; + + // ( tiles: $sources:expr $(, $tag:tt : $val:expr)* $(,)? ) => { + // $crate::utilejson! { + // + // tiles: $sources + // + // $( $tag : $val , )* + // } + // }; + // +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use std::collections::BTreeMap; + + use tilejson::TileJSON; + + use crate::utilejson::{Terrain, UTileJSON}; + + #[test] + pub fn test_utilejson_stringify() { + // Example TileJSON instance + let tile_json = TileJSON { + tilejson: "3.0.0".to_string(), + name: Some("Example TileJSON".to_string()), + scheme: None, + template: None, + version: None, + tiles: vec!["https://example.com/{z}/{x}/{y}.png".to_string()], + vector_layers: None, + attribution: None, + bounds: None, + center: None, + data: None, + description: None, + fillzoom: None, + grids: None, + legend: None, + maxzoom: None, + minzoom: None, + other: BTreeMap::default(), + }; + + // Create an instance of your wrapped struct + let utile_json = UTileJSON { + tj: tile_json, + terrain: Some(Terrain::Mapbox), + minzoom: 0, + maxzoom: 22, + id: None, + generator: None, + }; + let string = serde_json::to_string(&utile_json).unwrap(); + + let expected = "{\"tilejson\":\"3.0.0\",\"tiles\":[\"https://example.com/{z}/{x}/{y}.png\"],\"name\":\"Example TileJSON\",\"minzoom\":0,\"maxzoom\":22,\"terrain\":\"mapbox\"}"; + assert_eq!(string, expected); + let parsed = serde_json::from_str::(&string).unwrap(); + assert_eq!(parsed.terrain, Some(Terrain::Mapbox)); + // let tj_str = super::tilejson_stringify(&tj, None); + // assert_eq!(tj_str, "{\"tiles\":[]}"); + } + + #[test] + pub fn test_utilejson_macros() { + let utj = utilejson! { + tiles: vec!["https://example.com/{z}/{x}/{y}.png".to_string()], + minzoom: 0, + maxzoom: 30, + name: "Example TileJSON".to_string(), + scheme: "xyz".to_string(), + }; + + assert_eq!( + utj.tj.tiles, + vec!["https://example.com/{z}/{x}/{y}.png".to_string()] + ); + assert_eq!(utj.terrain, None); + assert_eq!(utj.minzoom, 0); + assert_eq!(utj.maxzoom, 30); + assert_eq!(utj.tj.name, Some("Example TileJSON".to_string())); + assert_eq!(utj.tj.scheme, Some("xyz".to_string())); + } +} diff --git a/crates/utiles/src/utilesqlite/mbtiles.rs b/crates/utiles/src/utilesqlite/mbtiles.rs index 69ddbb71..43852290 100644 --- a/crates/utiles/src/utilesqlite/mbtiles.rs +++ b/crates/utiles/src/utilesqlite/mbtiles.rs @@ -9,7 +9,7 @@ use tracing::{debug, error, warn}; use utiles_core::bbox::BBox; use utiles_core::constants::MBTILES_MAGIC_NUMBER; use utiles_core::tile_data_row::TileData; -use utiles_core::{yflip, LngLat, Tile, TileLike, UtilesCoreError}; +use utiles_core::{yflip, LngLat, Tile, TileLike}; use crate::errors::UtilesResult; use crate::mbt::query::{ @@ -19,8 +19,8 @@ use crate::mbt::query::{ query_mbtiles_type, }; use crate::mbt::{ - MbtMetadataRow, MbtType, MbtilesMetadataJson, MbtilesStats, MbtilesZoomStats, - MinZoomMaxZoom, + query_mbt_stats, MbtMetadataRow, MbtType, MbtilesMetadataJson, MbtilesStats, + MbtilesZoomStats, MinZoomMaxZoom, }; use crate::sqlite::RusqliteResult; use crate::sqlite::{ @@ -28,9 +28,9 @@ use crate::sqlite::{ pragma_table_list, query_db_fspath, Sqlike3, }; use crate::sqlite::{application_id_set, InsertStrategy}; +use crate::sqlite::{pathlike2dbpath, DbPath}; use crate::sqlite_utiles::add_ut_functions; use crate::utilejson::metadata2tilejson; -use crate::utilesqlite::dbpath::{pathlike2dbpath, DbPath}; use crate::UtilesError; pub struct Mbtiles { @@ -42,17 +42,21 @@ impl Sqlike3 for Mbtiles { fn conn(&self) -> &Connection { &self.conn } + + fn conn_mut(&mut self) -> &mut Connection { + &mut self.conn + } } impl Mbtiles { pub fn open>(path: P) -> UtilesResult { // if it is ':memory:' then open_in_memory let dbpath = pathlike2dbpath(path)?; - let conn_res = Connection::open(&dbpath.fspath); - match conn_res { - Ok(c) => Ok(Mbtiles { conn: c, dbpath }), - Err(e) => Err(UtilesError::RusqliteError(e)), - } + let conn_res = Connection::open(&dbpath.fspath)?; + Ok(Mbtiles { + conn: conn_res, + dbpath, + }) } pub fn open_with_flags>( @@ -142,17 +146,15 @@ impl Mbtiles { metadata_set_many(&self.conn, metadata) } - pub fn metadata_get(&self, name: &str) -> RusqliteResult> { + pub fn metadata_get(&self, name: &str) -> UtilesResult> { let rows = metadata_get(&self.conn, name)?; if rows.is_empty() { return Ok(None); } if rows.len() > 1 { error!( - "metadata has more than one row for name: {} - {}", - name, - serde_json::to_string(&rows) - .expect("metadata_get: error serializing metadata rows") + "metadata has more than one row for name: {} - {:?}", + name, rows, ); // return the first one let row = rows.first(); @@ -273,53 +275,7 @@ impl Mbtiles { } pub fn mbt_stats(&self, full: Option) -> UtilesResult { - let query_ti = std::time::Instant::now(); - let filesize = self.db_filesize()?; - let zoom_stats_full = full.unwrap_or(false) || filesize < 10_000_000_000; - debug!("Started zoom_stats query"); - let page_count = self.pragma_page_count()?; - let page_size = self.pragma_page_size()?; - let freelist_count = self.pragma_freelist_count()?; - // if the file is over 10gb and full is None or false just don't do the - // zoom_stats query that counts size... bc it is slow af - let zoom_stats = self.zoom_stats(zoom_stats_full)?; - debug!("zoom_stats: {:?}", zoom_stats); - let query_dt = query_ti.elapsed(); - debug!("Finished zoom_stats query in {:?}", query_dt); - let mbt_type = self.query_mbt_type()?; - if zoom_stats.is_empty() { - return Ok(MbtilesStats { - filesize, - mbtype: mbt_type, - page_count, - page_size, - freelist_count, - ntiles: 0, - minzoom: None, - maxzoom: None, - nzooms: 0, - zooms: vec![], - }); - } - - let minzoom = zoom_stats.iter().map(|r| r.zoom).min(); - let maxzoom = zoom_stats.iter().map(|r| r.zoom).max(); - let minzoom_u8: Option = minzoom - .map(|minzoom| minzoom.try_into().expect("Error converting minzoom to u8")); - let maxzoom_u8: Option = maxzoom - .map(|maxzoom| maxzoom.try_into().expect("Error converting maxzoom to u8")); - Ok(MbtilesStats { - ntiles: zoom_stats.iter().map(|r| r.ntiles).sum(), - filesize, - mbtype: mbt_type, - page_count, - page_size, - freelist_count, - minzoom: minzoom_u8, - maxzoom: maxzoom_u8, - nzooms: zoom_stats.len() as u32, - zooms: zoom_stats, - }) + query_mbt_stats(&self.conn, full) } pub fn zoom_stats(&self, full: bool) -> RusqliteResult> { @@ -735,6 +691,7 @@ pub fn init_mbtiles_normalized(conn: &mut Connection) -> RusqliteResult<()> { } pub fn init_mbtiles(conn: &mut Connection, mbt: &MbtType) -> UtilesResult<()> { + application_id_set(conn, MBTILES_MAGIC_NUMBER)?; let r: UtilesResult<()> = match mbt { MbtType::Flat => init_flat_mbtiles(conn).map_err(|e| e.into()), MbtType::Hash => init_mbtiles_hash(conn).map_err(|e| e.into()), @@ -751,28 +708,9 @@ pub fn create_mbtiles_file>( fspath: P, mbtype: &MbtType, ) -> UtilesResult { - let mut conn = Connection::open(fspath).map_err(|e| { - let emsg = format!("Error opening mbtiles file: {e}"); - UtilesCoreError::Unknown(emsg) - })?; - application_id_set(&conn, MBTILES_MAGIC_NUMBER)?; - match mbtype { - MbtType::Flat => { - init_flat_mbtiles(&mut conn)?; - Ok(conn) - } - MbtType::Hash => { - init_mbtiles_hash(&mut conn)?; - Ok(conn) - } - MbtType::Norm => { - init_mbtiles_normalized(&mut conn)?; - Ok(conn) - } - _ => Err(UtilesError::Unimplemented( - "create_mbtiles_file: only flat mbtiles is implemented".to_string(), - )), - } + let mut conn = Connection::open(fspath)?; + init_mbtiles(&mut conn, mbtype)?; + Ok(conn) } pub fn insert_tile_flat_mbtiles( @@ -790,8 +728,7 @@ pub fn insert_tiles_flat_mbtiles( tiles: &Vec, insert_strategy: Option, ) -> RusqliteResult { - let tx = conn.transaction().expect("Error creating transaction"); - + let tx = conn.transaction()?; let insert_strat = insert_strategy.unwrap_or_default(); let insert_clause = insert_strat.sql_prefix(); // TODO - use batch insert @@ -813,7 +750,7 @@ pub fn insert_tiles_flat_mbtiles( naff += r; } } - tx.commit().expect("Error committing transaction"); + tx.commit()?; Ok(naff) } @@ -1065,3 +1002,33 @@ pub fn query_distinct_tiletype(conn: &Connection) -> RusqliteResult> .collect::, rusqlite::Error>>()?; Ok(tile_format) } + +pub fn query_distinct_tilesize_zoom_limit( + conn: &Connection, + zoom: u32, + limit: u8, +) -> RusqliteResult> { + let mut stmt = conn.prepare_cached( + "SELECT DISTINCT ut_tilesize(tile_data) FROM tiles WHERE zoom_level=?1 LIMIT ?2", + )?; + + let tile_format: Vec = stmt + .query_map([zoom, u32::from(limit)], |row| row.get(0))? + .collect::, rusqlite::Error>>()?; + Ok(tile_format) +} + +pub fn query_distinct_tilesize_fast( + conn: &Connection, + min_max_zoom: MinZoomMaxZoom, +) -> RusqliteResult> { + let mut tile_types_set = HashSet::new(); + for z in min_max_zoom.minzoom..=min_max_zoom.maxzoom { + let a = query_distinct_tilesize_zoom_limit(conn, u32::from(z), 10)?; + for t in a { + tile_types_set.insert(t); + } + } + let tile_types_vec: Vec = tile_types_set.into_iter().collect(); + Ok(tile_types_vec) +} diff --git a/crates/utiles/src/utilesqlite/mbtiles_async.rs b/crates/utiles/src/utilesqlite/mbtiles_async.rs index 5f8868d5..5b70c333 100644 --- a/crates/utiles/src/utilesqlite/mbtiles_async.rs +++ b/crates/utiles/src/utilesqlite/mbtiles_async.rs @@ -4,15 +4,17 @@ use tilejson::TileJSON; use utiles_core::{BBox, Tile, TileLike}; use crate::errors::UtilesResult; -use crate::mbt::{MbtMetadataRow, MbtType}; +use crate::mbt::{MbtMetadataRow, MbtType, MbtilesStats}; use crate::mbt::{MbtilesMetadataJson, MinZoomMaxZoom}; use crate::sqlite::RowsAffected; + #[async_trait] pub trait MbtilesAsync: Sized { fn filepath(&self) -> &str; fn filename(&self) -> &str; async fn register_utiles_sqlite_functions(&self) -> UtilesResult<()>; + async fn is_mbtiles_like(&self) -> UtilesResult; async fn is_mbtiles(&self) -> UtilesResult; async fn assert_mbtiles(&self) -> UtilesResult<()>; async fn magic_number(&self) -> UtilesResult; @@ -48,4 +50,6 @@ pub trait MbtilesAsync: Sized { // async fn detach(&self, dbname: &str) -> UtilesResult; async fn zxyify(&self) -> UtilesResult>; + + async fn mbt_stats(&self, full: Option) -> UtilesResult; } diff --git a/crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs b/crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs index 66486480..9b4b463b 100644 --- a/crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs +++ b/crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs @@ -2,13 +2,28 @@ use std::fmt; use std::fmt::Debug; use std::path::Path; +use async_sqlite::{ + Client, ClientBuilder, Error as AsyncSqliteError, Pool, PoolBuilder, +}; +use async_trait::async_trait; +use rusqlite::{Connection, OpenFlags}; +use tilejson::TileJSON; +use tracing::{debug, error, info, warn}; + +use utiles_core::BBox; + use crate::errors::UtilesResult; use crate::mbt::query::query_mbtiles_type; use crate::mbt::zxyify::zxyify; -use crate::mbt::{MbtMetadataRow, MbtType, MbtilesMetadataJson, MinZoomMaxZoom}; -use crate::sqlite::{journal_mode, magic_number, AsyncSqliteConn, RowsAffected}; +use crate::mbt::{ + query_mbt_stats, MbtMetadataRow, MbtType, MbtilesMetadataJson, MbtilesStats, + MinZoomMaxZoom, +}; +use crate::sqlite::{ + journal_mode, magic_number, AsyncSqliteConn, AsyncSqliteConnMut, RowsAffected, +}; +use crate::sqlite::{pathlike2dbpath, DbPath, DbPathTrait}; use crate::utilejson::metadata2tilejson; -use crate::utilesqlite::dbpath::{pathlike2dbpath, DbPath, DbPathTrait}; use crate::utilesqlite::mbtiles::{ add_functions, has_metadata_table_or_view, has_tiles_table_or_view, has_zoom_row_col_index, init_mbtiles, mbtiles_metadata, mbtiles_metadata_row, @@ -16,14 +31,6 @@ use crate::utilesqlite::mbtiles::{ }; use crate::utilesqlite::mbtiles_async::MbtilesAsync; use crate::UtilesError; -use async_sqlite::{ - Client, ClientBuilder, Error as AsyncSqliteError, Pool, PoolBuilder, -}; -use async_trait::async_trait; -use rusqlite::{Connection, OpenFlags}; -use tilejson::TileJSON; -use tracing::{debug, error, info, warn}; -use utiles_core::BBox; #[derive(Clone)] pub struct MbtilesAsyncSqliteClient { @@ -58,14 +65,6 @@ impl Debug for MbtilesAsyncSqliteClient { } } -#[async_trait] -pub trait AsyncSqlite: Send + Sync { - async fn conn(&self, func: F) -> Result - where - F: FnOnce(&Connection) -> Result + Send + 'static, - T: Send + 'static; -} - #[async_trait] impl AsyncSqliteConn for MbtilesAsyncSqliteClient { async fn conn(&self, func: F) -> Result @@ -88,6 +87,28 @@ impl AsyncSqliteConn for MbtilesAsyncSqlitePool { } } +#[async_trait] +impl AsyncSqliteConnMut for MbtilesAsyncSqliteClient { + async fn conn_mut(&self, func: F) -> Result + where + F: FnOnce(&mut Connection) -> Result + Send + 'static, + T: Send + 'static, + { + self.client.conn_mut(func).await + } +} + +#[async_trait] +impl AsyncSqliteConnMut for MbtilesAsyncSqlitePool { + async fn conn_mut(&self, func: F) -> Result + where + F: FnOnce(&mut Connection) -> Result + Send + 'static, + T: Send + 'static, + { + self.pool.conn_mut(func).await + } +} + impl MbtilesAsyncSqliteClient { pub async fn new(dbpath: DbPath, client: Client) -> UtilesResult { let mbtype = client.conn(query_mbtiles_type).await?; @@ -105,7 +126,8 @@ impl MbtilesAsyncSqliteClient { let mbtype = mbtype.unwrap_or(MbtType::Flat); // make sure the path don't exist let dbpath = pathlike2dbpath(path)?; - if dbpath.fspath_exists() { + + if dbpath.fspath_exists_async().await { Err(UtilesError::PathExistsError(dbpath.fspath)) } else { debug!("Creating new mbtiles file with client: {}", dbpath); @@ -119,9 +141,7 @@ impl MbtilesAsyncSqliteClient { ) .open() .await?; - debug!("db-type is: {:?}", mbtype); - client .conn_mut(move |conn| { init_mbtiles(conn, &mbtype) @@ -183,9 +203,6 @@ impl MbtilesAsyncSqliteClient { Ok(jm) } } - -// impl Client -// pub async fn conn(&self, func: F) -> Result where F: FnOnce(&Connection) -> Result + Send + 'static, T: Send + 'static, impl MbtilesAsyncSqlitePool { pub async fn new(dbpath: DbPath, pool: Pool) -> UtilesResult { let mbtype = pool.conn(query_mbtiles_type).await?; @@ -270,8 +287,7 @@ where Ok(r) } - #[tracing::instrument] - async fn is_mbtiles(&self) -> UtilesResult { + async fn is_mbtiles_like(&self) -> UtilesResult { let has_metadata_table_or_view = self.conn(has_metadata_table_or_view).await?; debug!("has-metadata-table-or-view: {}", has_metadata_table_or_view); let has_tiles_table_or_view = self.conn(has_tiles_table_or_view).await?; @@ -280,28 +296,38 @@ where debug!("Not a mbtiles file: {}", self.filepath()); return Ok(false); } + Ok(true) + } + + async fn mbt_stats(&self, full: Option) -> UtilesResult { + self.conn(move |conn| { + let r = query_mbt_stats(conn, full); + Ok(r) + }) + .await? + } + + async fn is_mbtiles(&self) -> UtilesResult { + let is_mbtiles_like = self.is_mbtiles_like().await?; + if !is_mbtiles_like { + return Ok(false); + } // assert tiles is not empty let tiles_is_empty = self .conn(tiles_is_empty) .await - .map_err(UtilesError::AsyncSqliteError); - if let Ok(true) = tiles_is_empty { - debug!("Empty tiles table: {}", self.filepath()); - return Ok(false); - } - if let Err(e) = tiles_is_empty { - error!("Error checking if tiles table is empty: {}", e); - return Err(e); + .map_err(UtilesError::AsyncSqliteError)?; + if tiles_is_empty { + Ok(false) + } else { + let has_zoom_row_col_index = self.conn(has_zoom_row_col_index).await?; + debug!( + target: "is-mbtiles", + "has_zoom_row_col_index: {}", + has_zoom_row_col_index, + ); + Ok(has_zoom_row_col_index) } - - let has_zoom_row_col_index = self.conn(has_zoom_row_col_index).await?; - - debug!( - target: "is-mbtiles", - "has_zoom_row_col_index: {}", - has_zoom_row_col_index, - ); - Ok(has_zoom_row_col_index) } async fn assert_mbtiles(&self) -> UtilesResult<()> { @@ -522,110 +548,3 @@ where Ok(r) } } - -// ============================================================= -// ============================================================= -// NON GENERIC IMPLEMENTATION -// ============================================================= -// ============================================================= - -// #[async_trait] -// impl MbtilesAsync for MbtilesAsyncSqliteClient { -// fn filepath(&self) -> &str { -// &self.dbpath.fspath -// } -// -// fn filename(&self) -> &str { -// &self.dbpath.filename -// } -// -// -// async fn magic_number(&self) -> UtilesResult { -// self.client -// .conn(magic_number) -// .await -// .map_err(UtilesError::AsyncSqliteError) -// } -// -// async fn tilejson(&self) -> Result> { -// let metadata = self.metadata_rows().await?; -// let tj = metadata2tilejson(metadata); -// match tj { -// Ok(t) => Ok(t), -// Err(e) => { -// error!("Error parsing metadata to TileJSON: {}", e); -// Err(e) -// } -// } -// } -// -// async fn metadata_rows(&self) -> UtilesResult> { -// self.client -// .conn(mbtiles_metadata) -// .await -// .map_err(UtilesError::AsyncSqliteError) -// } -// -// async fn query_zxy(&self, z: u8, x: u32, y: u32) -> UtilesResult>> { -// self.client -// .conn(move |conn| query_zxy(conn, z, x, y)) -// .await -// .map_err(UtilesError::AsyncSqliteError) -// } -// } -// -// #[async_trait] -// impl MbtilesAsync for MbtilesAsyncSqlitePool { -// fn filepath(&self) -> &str { -// &self.dbpath.fspath -// } -// -// fn filename(&self) -> &str { -// &self.dbpath.filename -// } -// -// // async fn open(path: &str) -> UtilesResult { -// // let pool = PoolBuilder::new() -// // .path(path) -// // .journal_mode(JournalMode::Wal) -// // .open() -// // .await?; -// // Ok(MbtilesAsyncSqlitePool { -// // pool, -// // dbpath: DbPath::new(path), -// // }) -// // } -// -// async fn magic_number(&self) -> UtilesResult { -// self.pool -// .conn(magic_number) -// .await -// .map_err(UtilesError::AsyncSqliteError) -// } -// -// async fn tilejson(&self) -> Result> { -// let metadata = self.metadata_rows().await?; -// let tj = metadata2tilejson(metadata); -// match tj { -// Ok(t) => Ok(t), -// Err(e) => { -// error!("Error parsing metadata to TileJSON: {}", e); -// Err(e) -// } -// } -// } -// -// async fn metadata_rows(&self) -> UtilesResult> { -// self.pool -// .conn(mbtiles_metadata) -// .await -// .map_err(UtilesError::AsyncSqliteError) -// } -// -// async fn query_zxy(&self, z: u8, x: u32, y: u32) -> UtilesResult>> { -// self.pool -// .conn(move |conn| query_zxy(conn, z, x, y)) -// .await -// .map_err(UtilesError::AsyncSqliteError) -// } -// } diff --git a/crates/utiles/src/utilesqlite/mod.rs b/crates/utiles/src/utilesqlite/mod.rs index 876c4c91..4d9afbb9 100644 --- a/crates/utiles/src/utilesqlite/mod.rs +++ b/crates/utiles/src/utilesqlite/mod.rs @@ -1,7 +1,6 @@ // allow dead code in this module // #![allow(dead_code)] -mod dbpath; pub mod mbtiles; pub mod mbtiles_async; pub mod mbtiles_async_sqlite; diff --git a/dprint.json b/dprint.json new file mode 100644 index 00000000..aa6dd3e4 --- /dev/null +++ b/dprint.json @@ -0,0 +1,37 @@ +{ + "typescript": {}, + "json": { + "indentWidth": 2, + "useTabs": false, + "associations": [".prettierrc"] + }, + "markdown": {}, + "toml": {}, + "dockerfile": {}, + "ruff": {}, + "includes": [ + "**/*.{ts,tsx,mts,cts,cjs,mjs,js,jsx,json,md,toml,Dockerfile,dockerfile}", + "./**/{dockerfile,Dockerfile}" + ], + "excludes": [ + "**/*.min.{js,cjs,mjs,ts,mts,cts,jsx,tsx,json,geojson,lock}", + "**/*-lock.json", + "**/.next", + "**/.venv", + "**/build", + "**/coverage", + "**/dist", + "**/node_modules", + "**/temp", + "**/tmp", + "**/target" + ], + "plugins": [ + "https://plugins.dprint.dev/typescript-0.91.3.wasm", + "https://plugins.dprint.dev/json-0.19.3.wasm", + "https://plugins.dprint.dev/markdown-0.17.1.wasm", + "https://plugins.dprint.dev/toml-0.6.2.wasm", + "https://plugins.dprint.dev/dockerfile-0.3.2.wasm", + "https://plugins.dprint.dev/ruff-0.3.9.wasm" + ] +} diff --git a/justfile b/justfile index 4d377d2c..1a078943 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ pyut := "utiles-pyo3" pyut_manifest := pyut / "Cargo.toml" pyut_pyproject_toml := pyut / "pyproject.toml" -dev: develop test +dev: fmt develop pytest cargo-test develop: cd {{pyut}} @@ -23,9 +23,9 @@ dev-rel: cd {{pyut}} maturin develop --release -m {{pyut_manifest}} -test: +pytest: cd {{pyut}} - pytest --benchmark-disable --config-file={{pyut_pyproject_toml}} {{pyut}} + pytest --benchmark-disable -n 4 --config-file={{pyut_pyproject_toml}} {{pyut}} test-release: build-release cd {{pyut}} @@ -68,6 +68,10 @@ ruffix: clippy: cargo clippy + +clippy-fix: + cargo clippy --all-targets --all-features --fix --allow-dirty -- -D warnings + lintpy: ruff mypy lintrs: clippy diff --git a/utiles-pyo3/Cargo.toml b/utiles-pyo3/Cargo.toml index 600283d9..0ffe478a 100644 --- a/utiles-pyo3/Cargo.toml +++ b/utiles-pyo3/Cargo.toml @@ -1,22 +1,22 @@ [package] name = "pyutiles" -include = ["src/**/*", "Cargo.toml", "LICENSE", "README.md"] version.workspace = true +authors.workspace = true edition.workspace = true +include = ["src/**/*", "Cargo.toml", "LICENSE", "README.md"] license.workspace = true -authors.workspace = true [lib] name = "libutiles" crate-type = ["cdylib"] [dependencies] -utiles = { path = "../crates/utiles" } pyo3 = { workspace = true, features = ["experimental-async"] } serde.workspace = true serde_json.workspace = true -tracing.workspace = true size.workspace = true +tracing.workspace = true +utiles = { path = "../crates/utiles" } [dev-dependencies] pyo3 = { workspace = true, features = ["auto-initialize"] } diff --git a/utiles-pyo3/README.md b/utiles-pyo3/README.md index 232fd938..1707be83 100644 --- a/utiles-pyo3/README.md +++ b/utiles-pyo3/README.md @@ -137,11 +137,10 @@ test_ul_bench[mercantile-(486, 332, 20)] 1,099.9938 (5.38) 107,300.0021 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ``` - ## TODO: -- [X] benchmark against mercantile -- [X] Re-write cli in rust with clap +- [x] benchmark against mercantile +- [x] Re-write cli in rust with clap - **Maybe:** - - [] Mbtiles support?? - - [] Reading/writing mvt files? + - [] Mbtiles support?? + - [] Reading/writing mvt files? diff --git a/utiles-pyo3/pyproject.toml b/utiles-pyo3/pyproject.toml index 749d3835..8e6dd133 100644 --- a/utiles-pyo3/pyproject.toml +++ b/utiles-pyo3/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] @@ -7,21 +7,21 @@ name = "utiles" description = "utiles = (utils + tiles) * rust" requires-python = ">=3.8" classifiers = [ - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: Python", - "Programming Language :: Rust", - "Typing :: Typed", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: Python", + "Programming Language :: Rust", + "Typing :: Typed", ] authors = [{ name = "jesse rubin", email = "jessekrubin@gmail.com" }] maintainers = [{ name = "jesse rubin", email = "jessekrubin@gmail.com" }] @@ -51,124 +51,125 @@ module-name = "utiles._utiles" [tool.pytest.ini_options] testpaths = [ - "tests", + "tests", ] addopts = [ - "--doctest-modules", - # "--benchmark-disable", + "--doctest-modules", + # "--benchmark-disable", ] norecursedirs = [ - ".git", - ".nox", - ".pytest_cache", - ".venv", - "build", - "dist", - "scratch", - "node_modules", - "venv", - "*.bak", - "*.egg-info", - "*.egg", - ".*", - "target", - "utiles-cli", # in testing I write out a bunch of stuff to this dir... + ".git", + ".nox", + ".pytest_cache", + ".venv", + "build", + "dist", + "scratch", + "node_modules", + "venv", + "*.bak", + "*.egg-info", + "*.egg", + ".*", + "target", + "utiles-cli", # in testing I write out a bunch of stuff to this dir... ] markers = [ - "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "bench" + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "bench", ] - -[tool.black] -target-version = ["py38"] -line-length = 88 - [tool.ruff] target-version = "py38" line-length = 88 include = [ - "python/utiles/**/*.{py,pyi}", - "tests/**/*.{py,pyi}", - "bench/**/*.{py,pyi}" + "python/utiles/**/*.{py,pyi}", + "tests/**/*.{py,pyi}", + "bench/**/*.{py,pyi}", ] exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".mypy_cache", - ".nox", - ".pants.d", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "venv", + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", ] [tool.ruff.lint] select = [ - "A", - "ARG", - "B", - "C", - "DTZ", - "E", - "EM", - "F", - # "FBT", - "I", - "ICN", - "N", - "PLC", - #"ERA001", - "PLE", - "PLR", - "PLW", - "Q", - "RUF", - "S", - "T", - "TID", - "UP", - "W", - "YTT", + "A", + "ARG", + "B", + "C", + "DTZ", + "E", + "EM", + "F", + # "FBT", + "I", + "ICN", + "N", + "PLC", + # "ERA001", + "PLE", + "PLR", + "PLW", + "Q", + "RUF", + "S", + "T", + "TID", + "UP", + "W", + "YTT", ] ignore = [ - "TID252", - "A003", - # Allow non-abstract empty methods in abstract base classes - "B027", - # Allow boolean positional values in function calls, like `dict.get(... True)` - "FBT003", - # Ignore checks for possible passwords - "S105", "S106", "S107", - # Ignore complexity - "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", - # shadowing builtins - "A002", - "E501", # line length - # type annotations union - "UP007", - # todo figure out if this is needed - "UP006", - # magic value cmp super annoying - "PLR2004", - "PLW0120", + "TID252", + "A003", + # Allow non-abstract empty methods in abstract base classes + "B027", + # Allow boolean positional values in function calls, like `dict.get(... True)` + "FBT003", + # Ignore checks for possible passwords + "S105", + "S106", + "S107", + # Ignore complexity + "C901", + "PLR0911", + "PLR0912", + "PLR0913", + "PLR0915", + # shadowing builtins + "A002", + "E501", # line length + # type annotations union + "UP007", + # todo figure out if this is needed + "UP006", + # magic value cmp super annoying + "PLR2004", + "PLW0120", ] unfixable = [ - # Don't touch unused imports - "F401", + # Don't touch unused imports + "F401", ] [tool.ruff.lint.isort] @@ -190,7 +191,7 @@ source_pkgs = ["utiles", "tests"] branch = true parallel = true omit = [ - "python/utiles/__about__.py", + "python/utiles/__about__.py", ] [tool.coverage.paths] @@ -199,64 +200,11 @@ tests = ["tests", "*/utiles/tests"] [tool.coverage.report] exclude_lines = [ - "no cov", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", ] [tool.mypy] strict = true ignore_missing_imports = true - -# ============================================================================= -# [tool.hatch.envs.default] -# dependencies = [ -# "coverage[toml]>=6.5", -# "mercantile", -# "click", -# "typing_extensions", -# "tomli", -# "hypothesis", -# "maturin", -# "pytest", -# "pytest-benchmark", -# "pytest-cov", -# ] - -# [tool.hatch.envs.default.scripts] -# test = "pytest {args:tests}" -# test-cov = "coverage run -m pytest {args:tests}" -# cov-report = [ -# "- coverage combine", -# "coverage report", -# ] -# cov = [ -# "test-cov", -# "cov-report", -# ] - -# [[tool.hatch.envs.all.matrix]] -# python = ["3.8", "3.9", "3.10", "3.11", "3.12"] - -# [tool.hatch.envs.lint] -# detached = true -# dependencies = [ -# "black>=23.1.0", -# "mypy>=1.0.0", -# "ruff>=0.0.265", -# ] -# [tool.hatch.envs.lint.scripts] -# typing = "mypy --install-types --non-interactive {args:utiles tests}" -# style = [ -# "ruff {args:.}", -# "black --check --diff {args:.}", -# ] -# fmt = [ -# "black {args:.}", -# "ruff --fix {args:.}", -# "style", -# ] -# all = [ -# "style", -# "typing", -# ] diff --git a/utiles-pyo3/python/utiles/dev/testing.py b/utiles-pyo3/python/utiles/dev/testing.py index 00e4d78c..ea65ae70 100644 --- a/utiles-pyo3/python/utiles/dev/testing.py +++ b/utiles-pyo3/python/utiles/dev/testing.py @@ -6,9 +6,10 @@ from orjson import loads as json_loads except ImportError: from json import loads as json_loads - import sys from dataclasses import dataclass +from pathlib import Path +from sqlite3 import connect from subprocess import CompletedProcess, run from time import time_ns from typing import Any @@ -118,3 +119,15 @@ def run_cli( if res.completed_process.returncode != 0: res.echo() return res + + +def query_metadata_rows( + dbpath: str | Path, +) -> list[dict[str, Any]]: + """Query metadata rows""" + + with connect(dbpath) as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM metadata;") + rows = cursor.fetchall() + return [dict(zip((d[0] for d in cursor.description), row)) for row in rows] diff --git a/utiles-pyo3/requirements/dev.in b/utiles-pyo3/requirements/dev.in index 7395ca7a..6cf18e34 100644 --- a/utiles-pyo3/requirements/dev.in +++ b/utiles-pyo3/requirements/dev.in @@ -7,6 +7,7 @@ pmtiles pytest pytest-benchmark pytest-cov +pytest-xdist ruff tomli typing_extensions diff --git a/utiles-pyo3/requirements/dev.txt b/utiles-pyo3/requirements/dev.txt index a9b56d50..9f87b331 100644 --- a/utiles-pyo3/requirements/dev.txt +++ b/utiles-pyo3/requirements/dev.txt @@ -13,17 +13,19 @@ colorama==0.4.6 # pytest colorlog==6.8.2 # via nox -coverage==7.5.4 +coverage==7.6.0 # via pytest-cov distlib==0.3.8 # via virtualenv +execnet==2.1.1 + # via pytest-xdist filelock==3.15.4 # via virtualenv -hypothesis==6.104.1 +hypothesis==6.108.2 # via -r dev.in iniconfig==2.0.0 # via pytest -maturin==1.6.0 +maturin==1.7.0 # via -r dev.in mercantile==1.2.1 # via -r dev.in @@ -50,11 +52,14 @@ pytest==8.2.2 # -r dev.in # pytest-benchmark # pytest-cov + # pytest-xdist pytest-benchmark==4.0.0 # via -r dev.in pytest-cov==5.0.0 # via -r dev.in -ruff==0.5.0 +pytest-xdist==3.6.1 + # via -r dev.in +ruff==0.5.2 # via -r dev.in sortedcontainers==2.4.0 # via hypothesis diff --git a/utiles-pyo3/src/cli.rs b/utiles-pyo3/src/cli.rs index 74f390ec..01c265ec 100644 --- a/utiles-pyo3/src/cli.rs +++ b/utiles-pyo3/src/cli.rs @@ -1,6 +1,7 @@ use pyo3::exceptions::PyException; use pyo3::{pyfunction, PyResult}; -use utiles::cli::cli_main_sync; + +use utiles::cli::{cli_main_sync, CliOpts}; #[pyfunction] #[pyo3(signature = (args = None))] @@ -32,7 +33,10 @@ pub fn ut_cli(args: Option>) -> PyResult { // Some(&|| { // py.check_signals().unwrap(); // }), - let rc = cli_main_sync(Some(utiles_argv)); + let rc = cli_main_sync(Some(CliOpts { + argv: Some(utiles_argv), + clid: Some("pyo3"), + })); match rc { Ok(_) => Ok(0), Err(e) => { diff --git a/utiles-pyo3/src/pyutiles/pytile.rs b/utiles-pyo3/src/pyutiles/pytile.rs index 06be7e56..e2610a63 100644 --- a/utiles-pyo3/src/pyutiles/pytile.rs +++ b/utiles-pyo3/src/pyutiles/pytile.rs @@ -11,9 +11,9 @@ use pyo3::{ PyResult, Python, }; use serde::Serialize; - use utiles::bbox::BBox; -use utiles::tile::Tile; +use utiles::projection::Projection; +use utiles::tile::{FeatureOptions, Tile}; use utiles::TileLike; use crate::pyutiles::pyiters::IntIterator; @@ -95,12 +95,12 @@ impl PyTile { Py::new(slf.py(), iter) } - #[pyo3(signature = (sep=None))] + #[pyo3(signature = (sep = None))] pub fn fmt_zxy(&self, sep: Option<&str>) -> String { self.xyz.fmt_zxy(sep) } - #[pyo3(signature = (ext="", sep=None))] + #[pyo3(signature = (ext = "", sep = None))] pub fn fmt_zxy_ext(&self, ext: &str, sep: Option<&str>) -> String { self.xyz.fmt_zxy_ext(ext, sep) } @@ -166,7 +166,7 @@ impl PyTile { } #[classmethod] - #[pyo3(signature = (lng, lat, zoom, truncate=None))] + #[pyo3(signature = (lng, lat, zoom, truncate = None))] pub fn from_lnglat_zoom( _cls: &Bound<'_, PyType>, lng: f64, @@ -358,12 +358,12 @@ impl PyTile { self.xyz.center().into() } - #[pyo3(signature = (n=None))] + #[pyo3(signature = (n = None))] pub fn parent(&self, n: Option) -> Self { self.xyz.parent(n).into() } - #[pyo3(signature = (zoom=None))] + #[pyo3(signature = (zoom = None))] pub fn children(&self, zoom: Option) -> Vec { let xyzs = self.xyz.children(zoom); xyzs.into_iter().map(Self::from).collect() @@ -381,107 +381,69 @@ impl PyTile { self.xyz.into() } - #[pyo3(signature = (fid=None, props=None, projected=None, buffer=None, precision=None))] + #[pyo3( + signature = (fid = None, props = None, projected = None, buffer = None, precision = None) + )] pub fn feature( &self, py: Python, - // tile: PyTileLike, - // (u32, u32, u8), fid: Option, props: Option>>, projected: Option, buffer: Option, precision: Option, ) -> PyResult> { - // Convert the arguments to Rust values - // let pytile: PyTile = tile.into(); - // let tile = pytile.tuple(); - let (x, y, z) = self.tuple(); - let fid = fid.unwrap_or_default(); - let props = props.unwrap_or_default(); - let projected = projected.unwrap_or_else(|| "geographic".to_string()); - let buffer = buffer.unwrap_or(0.0); - let precision = precision.unwrap_or(-1); - - // Compute the bounds - let (west, south, east, north) = utiles::bounds(x, y, z); - - // Handle projected coordinates - let (mut west, mut south, mut east, mut north) = match projected.as_str() { - "mercator" => { - // let (east_merc, north_merc) = utiles::xy(east, north, Some(false)); - let (west_merc, south_merc) = utiles::xy(west, south, None); - let (east_merc, north_merc) = utiles::xy(east, north, None); - (west_merc, south_merc, east_merc, north_merc) - } - _ => (west, south, east, north), + let projection = if let Some(projected) = projected { + Projection::try_from(projected) + .map_err(|e| PyErr::new::(format!("Error: {e}")))? + } else { + Projection::Geographic }; + let feature_opts = FeatureOptions { + buffer, + fid, + precision, + projection, + props: None, + }; + let tfeat = self + .xyz + .feature(&feature_opts) + .map_err(|e| PyErr::new::(format!("Error: {e}")))?; - // Apply buffer - west -= buffer; - south -= buffer; - east += buffer; - north += buffer; - - // Apply precision - if precision >= 0 { - let precision_factor = 10_f64.powi(precision); - west = (west * precision_factor).round() / precision_factor; - south = (south * precision_factor).round() / precision_factor; - east = (east * precision_factor).round() / precision_factor; - north = (north * precision_factor).round() / precision_factor; - } - - // Compute bbox and geometry - let bbox = [ - west.min(east), - south.min(north), - west.max(east), - south.max(north), - ]; - let geometry_coordinates = vec![vec![ - [west, south], - [west, north], - [east, north], - [east, south], - [west, south], - ]]; - + // feature that will become python object + let mut feature_dict = HashMap::new(); + let bbox_vec = vec![tfeat.bbox.0, tfeat.bbox.1, tfeat.bbox.2, tfeat.bbox.3]; let geometry_items = vec![ - ("type".to_string(), "Polygon".to_object(py)), + ("type".to_string(), tfeat.geometry.type_.to_object(py)), ( "coordinates".to_string(), - geometry_coordinates.to_object(py), + tfeat.geometry.coordinates.to_object(py), ), ] .into_iter() .collect::>(); - - // Create the feature dictionary - let xyz = format!("({x}, {y}, {z})").into_py(py); - let mut feature_dict = HashMap::new(); - feature_dict.insert("type".to_string(), "Feature".to_object(py)); - feature_dict.insert("bbox".to_string(), bbox.to_object(py)); - feature_dict.insert("id".to_string(), xyz.to_object(py)); - feature_dict.insert("geometry".to_string(), geometry_items.to_object(py)); - // Create the properties dictionary let mut properties_dict: HashMap> = HashMap::new(); - properties_dict - .insert("title".to_string(), format!("XYZ tile {xyz}").into_py(py)); - if !props.is_empty() { + let (x, y, z) = self.tuple(); + let xyz_tuple_string = format!("({x}, {y}, {z})").into_py(py); + + properties_dict.insert( + "title".to_string(), + format!("XYZ tile {xyz_tuple_string}").into_py(py), + ); + if let Some(props) = props { let props: PyResult)>> = props .into_iter() .map(|(k, v)| Ok((k, v.into_py(py)))) .collect(); properties_dict.extend(props?); } + feature_dict.insert("type".to_string(), "Feature".to_object(py)); + feature_dict.insert("bbox".to_string(), bbox_vec.to_object(py)); + feature_dict.insert("id".to_string(), tfeat.id.to_object(py)); + feature_dict.insert("geometry".to_string(), geometry_items.to_object(py)); feature_dict.insert("properties".to_string(), properties_dict.to_object(py)); - - // Add the feature id if provided - if !fid.is_empty() { - feature_dict.insert("id".to_string(), fid.to_object(py)); - } Ok(feature_dict) } } diff --git a/utiles-pyo3/tests/cli/test_copy_db.py b/utiles-pyo3/tests/cli/test_copy_db.py new file mode 100644 index 00000000..7cc1fad4 --- /dev/null +++ b/utiles-pyo3/tests/cli/test_copy_db.py @@ -0,0 +1,211 @@ +"""Utiles rust cli tests""" + +from pathlib import Path + +from utiles.dev.testing import run_cli as _run_cli + + +def _osm_standard_z0z4_mbtiles(test_data: Path) -> Path: + return test_data / "mbtiles" / "osm-standard.z0z4.mbtiles" + + +def _all_filepaths(dirpath: Path) -> list[str]: + return [str(f) for f in dirpath.rglob("*") if f.is_file()] + + +def test_copy_mbtiles(tmp_path: Path, test_data_root: Path, db_type: str) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + out_path = tmp_path / "copied.mbtiles" + copy_result = _run_cli( + ["cp", str(_mbtiles_filepath), str(out_path), "--dbtype", db_type] + ) + + assert copy_result.returncode == 0 + info_result = _run_cli(["info", str(out_path)]) + info_dict = info_result.parse_json + expected_key_values = {"ntiles": 341, "nzooms": 5, "minzoom": 0, "maxzoom": 4} + for k, v in expected_key_values.items(): + assert info_dict[k] == v + + assert db_type == info_dict["mbtype"] + + +def test_copy_mbtiles_zooms(tmp_path: Path, test_data_root: Path, db_type: str) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + out_path = tmp_path / "copied.mbtiles" + copy_result = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--minzoom", + "3", + ] + ) + + assert copy_result.returncode == 0 + info_result = _run_cli(["info", str(out_path)]) + info_dict = info_result.parse_json + expected_key_values = { + "ntiles": (((2 << (3 - 1)) ** 2) + ((2 << (4 - 1)) ** 2)), + "nzooms": 2, + "minzoom": 3, + "maxzoom": 4, + } + for k, v in expected_key_values.items(): + assert info_dict[k] == v + + assert db_type == info_dict["mbtype"] + + +def test_copy_mbtiles_conflict( + tmp_path: Path, test_data_root: Path, db_type: str +) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + out_path = tmp_path / "copied.mbtiles" + copy_result_a = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--minzoom", + "3", + ] + ) + assert copy_result_a.returncode == 0 + + # no specifying of the zooms... + copy_result_b = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + ] + ) + + assert copy_result_b.returncode != 0 + + +def test_copy_mbtiles_conflict_with_strategy( + tmp_path: Path, test_data_root: Path, db_type: str +) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + out_path = tmp_path / "copied.mbtiles" + copy_result_a = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--minzoom", + "3", + ] + ) + + assert copy_result_a.returncode == 0 + + # no specifying of the zooms... + copy_result_b = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--debug", + "--conflict", + "ignore", + ] + ) + assert copy_result_b.returncode == 0 + + +def test_copy_mbtiles_conflict_with_strategy_not_overlapping( + tmp_path: Path, test_data_root: Path, db_type: str +) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + out_path = tmp_path / "copied.mbtiles" + copy_result_a = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--minzoom", + "3", + ] + ) + + assert copy_result_a.returncode == 0 + + # no specifying of the zooms... + copy_result_b = _run_cli( + [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--debug", + "--maxzoom", + "2", + ] + ) + assert copy_result_b.returncode == 0 + + +def test_copy_mbtiles_bbox(tmp_path: Path, test_data_root: Path, db_type: str) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + out_path = tmp_path / "copied.mbtiles" + west_half_o_world_args = [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--minzoom", + "3", + "--maxzoom", + "4", + "--bbox", + "-180,-90,0,90", + ] + print(" ".join(west_half_o_world_args)) + copy_result_a = _run_cli(west_half_o_world_args) + + assert copy_result_a.returncode == 0 + + info_result = _run_cli(["info", str(out_path)]) + info_dict = info_result.parse_json + expected_ntiles_total = ((2 << (3 - 1)) ** 2) + ((2 << (4 - 1)) ** 2) + assert info_dict["ntiles"] == expected_ntiles_total // 2 + + # no specifying of the zooms... + east_half_o_world_args = [ + "cp", + str(_mbtiles_filepath), + str(out_path), + "--dbtype", + db_type, + "--minzoom", + "3", + "--maxzoom", + "4", + "--bbox", + "0,-90,180,90", + ] + copy_result_b = _run_cli(east_half_o_world_args) + assert copy_result_b.returncode == 0 + + info_result_final = _run_cli(["info", str(out_path)]) + info_dict_final = info_result_final.parse_json + print(info_dict_final) + assert info_dict_final["ntiles"] == expected_ntiles_total diff --git a/utiles-pyo3/tests/cli/test_copy.py b/utiles-pyo3/tests/cli/test_copy_pyramid.py similarity index 100% rename from utiles-pyo3/tests/cli/test_copy.py rename to utiles-pyo3/tests/cli/test_copy_pyramid.py diff --git a/utiles-pyo3/tests/cli/test_db.py b/utiles-pyo3/tests/cli/test_db.py index 02f8c355..551a5cd9 100644 --- a/utiles-pyo3/tests/cli/test_db.py +++ b/utiles-pyo3/tests/cli/test_db.py @@ -3,6 +3,8 @@ import json from pathlib import Path +from pytest import fixture + from utiles.dev.testing import run_cli as _run_cli @@ -39,6 +41,23 @@ def test_touch(tmp_path: Path, test_data_root: Path) -> None: assert parsed_data == expected_info_json +def test_touch_db_type(tmp_path: Path, db_type: str) -> None: + # make a new file + new_mbtiles = tmp_path / "new.mbtiles" + result = _run_cli(["touch", str(new_mbtiles), "--db-type", db_type]) + assert result.returncode == 0 + assert new_mbtiles.exists() + assert new_mbtiles.is_file() + assert new_mbtiles.suffix == ".mbtiles" + + result = _run_cli(["info", str(new_mbtiles)]) + assert result.returncode == 0 + result.print() + parsed_data = json.loads(result.stdout) + assert parsed_data["ntiles"] == 0 + assert parsed_data["mbtype"] == db_type + + def test_touch_page_size_512(tmp_path: Path) -> None: # make a new file new_mbtiles = tmp_path / "new.mbtiles" @@ -54,11 +73,11 @@ def test_touch_page_size_512(tmp_path: Path) -> None: parsed_data = json.loads(result.stdout) assert parsed_data["ntiles"] == 0 expected_info_json = { - "filesize": 2560, + "filesize": 2048, "mbtype": "flat", "ntiles": 0, "nzooms": 0, - "page_count": 5, + "page_count": 4, "page_size": 512, "freelist_count": 0, "minzoom": None, diff --git a/utiles-pyo3/tests/cli/test_mt.py b/utiles-pyo3/tests/cli/test_mt.py index 115ce72e..affa7199 100644 --- a/utiles-pyo3/tests/cli/test_mt.py +++ b/utiles-pyo3/tests/cli/test_mt.py @@ -29,7 +29,7 @@ def _run_cli_old( def test_rust_cli_help() -> None: res = _run_cli(["--help"]) - assert "(rust)" in res.stdout + assert "(rust)" in res.stdout or "(pyo3)" in res.stdout class TestTiles: diff --git a/utiles-pyo3/tests/cli/test_update.py b/utiles-pyo3/tests/cli/test_update.py new file mode 100644 index 00000000..47cde54d --- /dev/null +++ b/utiles-pyo3/tests/cli/test_update.py @@ -0,0 +1,52 @@ +"""Utiles rust cli tests""" + +import sqlite3 +from pathlib import Path +from shutil import copyfile + +from utiles.dev.testing import query_metadata_rows +from utiles.dev.testing import run_cli as _run_cli + + +def _osm_standard_z0z4_mbtiles(test_data: Path) -> Path: + return test_data / "mbtiles" / "osm-standard.z0z4.mbtiles" + + +def _all_filepaths(dirpath: Path) -> list[str]: + return [str(f) for f in dirpath.rglob("*") if f.is_file()] + + +def test_update_metadata(tmp_path: Path, test_data_root: Path) -> None: + _mbtiles_filepath = _osm_standard_z0z4_mbtiles(test_data_root) + # copy the mbt db file to the tmp_path + + dbpath = tmp_path / "test.mbtiles" + copyfile(_mbtiles_filepath, dbpath) + metadata_keys_to_delete = ["format", "minzoom", "maxzoom", "tilesize", "name"] + + with sqlite3.connect(dbpath) as conn: + cursor = conn.cursor() + for key in metadata_keys_to_delete: + cursor.execute(f"DELETE FROM metadata WHERE name = '{key}';") + conn.commit() + + # get the metadata from the new mbt + metadata_proc = _run_cli(["metadata", str(dbpath)]) + metadata = metadata_proc.parse_json + for key in metadata_keys_to_delete: + assert key not in metadata + + # run update on the new mbt + update_result = _run_cli(["update", str(dbpath), "--debug"]) + update_result.echo() + + assert update_result.returncode == 0 + + metadata_proc_updated = _run_cli(["metadata", "--obj", str(dbpath)]) + metadata_updated = metadata_proc_updated.parse_json + print(metadata_updated) + assert metadata_updated["format"] == "png" + assert metadata_updated["minzoom"] == 0 + assert metadata_updated["maxzoom"] == 4 + # assert metadata_updated["tilesize"] == 256 + # assert metadata_updated["name"] == "osm-standard.z0z4" diff --git a/utiles-pyo3/tests/conftest.py b/utiles-pyo3/tests/conftest.py index 1a44b124..606e1753 100644 --- a/utiles-pyo3/tests/conftest.py +++ b/utiles-pyo3/tests/conftest.py @@ -25,3 +25,9 @@ def repo_root() -> Path: @pytest.fixture def test_data_root(repo_root: Path) -> Path: return repo_root / "test-data" + + +@pytest.fixture(params=["flat", "hash", "norm"]) +def db_type(request: pytest.FixtureRequest) -> str: + """Fixture for testing different db/schema types""" + return request.param