utiles v0.30 (#21)

* pre ci

* fmt

* fns!

* tmp

* aha jesus dogs

* parse bbox of size 4+

* fix profiles and add help messages

* add cargo depbot

* aha fixed click and not req for utiles anymore!

* minor fixes

* more parsing

* parsing dem ints

* updated python deps

* row major!

* row major index

* linting

* utiles v 3

* fixed missing metadata table when doing copy..........

.github/dependabot.yml

@@ -5,3 +5,9 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10

Cargo.toml

@@ -11,7 +11,7 @@ members = [
 
 [workspace.package]
 edition = "2021"
-version = "0.2.0"
+version = "0.3.0"
 homepage = "https://github.com/jessekrubin/utiles"
 documentation = "https://github.com/jessekrubin/utiles"
 repository = "https://github.com/jessekrubin/utiles"

crates/utiles-cli/src/args.rs

@@ -178,9 +178,11 @@ pub enum Commands {
         lnglat: LngLat,
     },
 
-    // ========================================================================
-    // TILE CLI UTILS - MERCANTILE LIKE CLI
-    // ========================================================================
+    /*
+    ========================================================================
+    TILE CLI UTILS - MERCANTILE LIKE CLI
+    ========================================================================
+    */
     /// Echo the Web Mercator tile at ZOOM level bounding GeoJSON [west, south,
     /// east, north] bounding boxes, features, or collections read from stdin.
     ///
@@ -199,21 +201,76 @@ pub enum Commands {
     )]
     BoundingTile(TileFmtArgs),
 
+    ///  Converts tiles given as [x, y, z] and/or quadkeys to/from the other format
+    ///
+    /// Input may be a compact newline-delimited sequences of JSON or a pretty-
+    /// printed ASCII RS-delimited sequence of JSON (like
+    /// https://tools.ietf.org/html/rfc8142 and
+    /// https://tools.ietf.org/html/rfc7159).
+    ///
+    /// Examples:
+    ///
+    /// echo "[486, 332, 10]" | utiles quadkey
+    /// 0313102310
+    ///
+    /// echo "0313102310" | utiles quadkey
+    /// [486, 332, 10]
     #[command(name = "quadkey", visible_alias = "qk", about = "Convert to/from quadkey(s)", long_about = None)]
     Quadkey(TileFmtArgs),
 
-    #[command(name = "tiles", about = "Echo tiles of bbox", long_about = None)]
+    /// Echos web-mercator tiles at zoom level intersecting given geojson-bbox [west, south,
+    /// east, north], geojson-features, or geojson-collections read from stdin.
+    ///
+    /// Output format is a JSON `[x, y, z]` array by default; use --obj to output a
+    /// JSON object `{x: x, y: y, z: z}`.
+    ///
+    /// Input may be a compact newline-delimited sequences of JSON or a pretty-
+    /// printed ASCII RS-delimited sequence of JSON (like
+    /// https://tools.ietf.org/html/rfc8142 and
+    /// https://tools.ietf.org/html/rfc7159).
+    ///
+    /// Example:
+    ///
+    /// $ echo "[-105.05, 39.95, -105, 40]" | utiles tiles 12
+    /// [852, 1550, 12]
+    /// [852, 1551, 12]
+    /// [853, 1550, 12]
+    /// [853, 1551, 12]
+    #[command(name = "tiles", about = "Echo tiles of bbox")]
     Tiles(TilesArgs),
 
+    /// Converts tiles to/from xyz ([x, y, z]) and/or pmtile-id format(s)
+    ///
+    /// Input may be a compact newline-delimited sequences of JSON or a pretty-
+    /// printed ASCII RS-delimited sequence of JSON (like
+    /// https://tools.ietf.org/html/rfc8142 and
+    /// https://tools.ietf.org/html/rfc7159).
+    ///
+    /// Examples:
+    ///
+    /// echo "[486, 332, 10]" | utiles pmtileid
+    /// 506307
+    ///
+    /// echo "506307" | utiles pmtileid
+    /// [486, 332, 10]
     #[command(name = "pmtileid", visible_alias = "pmid", about = "Convert to/from pmtile id(s)", long_about = None)]
     Pmtileid(TileFmtArgs),
 
+    /// Echo the neighbor tiles for input tiles
+    ///
+    /// Input may be a compact newline-delimited sequences of JSON or a pretty-
+    /// printed ASCII RS-delimited sequence of JSON (like
+    /// https://tools.ietf.org/html/rfc8142 and
+    /// https://tools.ietf.org/html/rfc7159).
+    ///
     #[command(name = "neighbors", about = "Echo neighbors of tile(s)", long_about = None)]
     Neighbors(TileFmtArgs),
 
+    /// Echo children tiles of input tiles
     #[command(name = "children", about = "Echo children of tile(s)", long_about = None)]
     Children(ParentChildrenArgs),
 
+    /// Echo parent tile of input tiles
     #[command(name = "parent", about = "Echo parent of tile(s)", long_about = None)]
     Parent(ParentChildrenArgs),
 

crates/utiles-cli/src/cli.rs

@@ -47,10 +47,9 @@ fn init_tracing(log_config: LogConfig) {
 #[allow(clippy::unused_async)]
 pub async fn cli_main(argv: Option<Vec<String>>, loop_fn: Option<&dyn Fn()>) -> u8 {
     // print args
-    let argv = match argv {
-        Some(argv) => argv,
-        None => std::env::args().collect::<Vec<_>>(),
-    };
+    let argv = argv.unwrap_or_else(|| std::env::args().collect::<Vec<_>>());
+
+    // set caller if provided
     let args = Cli::parse_from(&argv);
 
     // if the command is "dev" init tracing w/ debug
@@ -60,11 +59,10 @@ pub async fn cli_main(argv: Option<Vec<String>>, loop_fn: Option<&dyn Fn()>) ->
             json: args.log_json,
         });
     } else {
-        let log_config = LogConfig {
+        init_tracing(LogConfig {
             debug: args.debug,
             json: args.log_json,
-        };
-        init_tracing(log_config);
+        });
     }
 
     debug!("args: {:?}", std::env::args().collect::<Vec<_>>());

crates/utiles-cli/src/commands/copy.rs

@@ -13,7 +13,7 @@ use walkdir::WalkDir;
 use utiles::bbox::BBox;
 use utiles::mbtiles::{MbtTileRow, MbtilesMetadataRow};
 use utiles::tile_data_row::TileData;
-use utiles::{flipy, tile_ranges, Tile, TileLike};
+use utiles::{tile_ranges, Tile, TileLike};
 use utilesqlite::Mbtiles;
 
 use crate::args::CopyArgs;
@@ -60,9 +60,11 @@ impl TilesFsWriter {
     }
 
     pub async fn write_tile(&self, tile: MbtTileRow) {
+        // HERE YOU NEED TO FLIP THE THING JESSE
         let filepath = self.dirpath(tile.z(), tile.x()).join(format!(
             "{}.{}",
-            flipy(tile.y(), tile.z()),
+            // flipy(tile.y(), tile.z()),
+            tile.y(),
             tile.extension()
         ));
         // debug!("filepath: {:?}", filepath);
@@ -183,23 +185,27 @@ async fn copy_mbtiles2fs(mbtiles: String, output_dir: String, cfg: CopyConfig) {
 
     let where_clause = cfg.sql_where(
         // Some(zoom_levels_for_where)
+        // flip happens here maybe
         None,
     );
     let start_time = std::time::Instant::now();
 
+    let count_query = &"SELECT count(*) FROM tiles".to_string();
     let total_tiles: u32 = mbt
         .conn()
-        .query_row(
-            &format!("SELECT count(*) FROM tiles {where_clause}"),
-            [],
-            |row| row.get(0),
-        )
+        .query_row(count_query, [], |row| row.get(0))
         .unwrap();
 
-    info!("finna write {total_tiles:?} from {mbtiles:?} to {output_dir:?}");
+    debug!("total_tiles: {:?}", total_tiles);
+
+    info!("# tiles: {total_tiles:?} ~ {mbtiles:?} => {output_dir:?}");
     let c = mbt.conn();
 
-    let metadata_vec = mbt.metadata().unwrap();
+    let res_metadata_vec = mbt.metadata();
+    let metadata_vec = res_metadata_vec.unwrap_or_else(|e| {
+        warn!("e: {e:?}");
+        vec![]
+    });
     let metadata_str = serde_json::to_string_pretty(&metadata_vec).unwrap();
     // ensure output_dir exists
     fs::create_dir_all(&output_dir).await.unwrap();
@@ -253,10 +259,8 @@ async fn copy_mbtiles2fs(mbtiles: String, output_dir: String, cfg: CopyConfig) {
         .query_map([], |row| {
             let zoom_level: u8 = row.get(0)?;
             let tile_column: u32 = row.get(1)?;
-
             let tile_row: u32 = row.get(2)?;
             let tile_data: Vec<u8> = row.get(3)?;
-
             let r = MbtTileRow::new(zoom_level, tile_column, tile_row, tile_data);
             Ok(r)
         })
@@ -269,8 +273,6 @@ async fn copy_mbtiles2fs(mbtiles: String, output_dir: String, cfg: CopyConfig) {
         .for_each_concurrent(0, |tile| async {
             match tile {
                 Ok(tile) => {
-                    let _t =
-                        Tile::new(tile.tile_column, tile.tile_row, tile.zoom_level);
                     twriter.write_tile(tile).await;
                 }
                 Err(e) => {

crates/utiles-cli/src/commands/tile_stream_cmds.rs

@@ -36,10 +36,13 @@ pub fn pmtileid_main(args: TileFmtArgs) {
     let lines = stdinterator_filter::stdin_filtered(args.inargs.input);
     for line in lines {
         // if the line bgins w '[' treat as tile
-        let lstr = line.unwrap();
+        let lstr = line
+            .unwrap() // remove the `"` and `'` chars from the beginning and end of the line
+            .trim_matches(|c| c == '"' || c == '\'')
+            .to_string();
         if lstr.starts_with('[') {
             // treat as tile
-            let tile = Tile::from_json_arr(&lstr).unwrap();
+            let tile = Tile::from_json(&lstr).unwrap();
             println!("{}", tile.pmtileid());
         } else {
             // treat as pmtileid
@@ -65,7 +68,7 @@ pub fn quadkey_main(args: TileFmtArgs) {
         match first_char {
             '[' | '{' => {
                 // treat as tile
-                let tile = Tile::from_json_arr(&lstr).unwrap();
+                let tile = Tile::from_json(&lstr).unwrap();
                 println!("{}", tile.quadkey());
             }
             _ => {

crates/utiles-cli/src/commands/tiles.rs

@@ -35,6 +35,8 @@ impl TileStringFormatter for TileFmt {
 pub fn tiles_main(args: TilesArgs, loop_fn: Option<&dyn Fn()>) {
     let lines = stdin_filtered(args.inargs.input);
     let mut stdout = io::stdout();
+    let lock = stdout.lock();
+    let mut buf = std::io::BufWriter::with_capacity(32 * 1024, lock);
     let tiles = lines
         .map(|l| {
             let s = l.unwrap();
@@ -57,7 +59,10 @@ pub fn tiles_main(args: TilesArgs, loop_fn: Option<&dyn Fn()>) {
 
     let rs = if args.fmtopts.seq { "\x1e\n" } else { "" };
     for (i, tile) in tiles {
-        writeln!(stdout, "{}{}", rs, tile_fmt.format_tile(&tile)).unwrap();
+        let tile_str = tile_fmt.format_tile(&tile);
+        let out_str = format!("{rs}{tile_str}\n");
+        buf.write_all(out_str.as_bytes()).unwrap();
+        // writeln!(stdout, "{}{}", rs, tile_fmt.format_tile(&tile)).unwrap();
         // call loop_fn if it's defined every 1000 iterations for signal break
         if i % 1024 == 0 {
             stdout.flush().unwrap();

crates/utiles-dev/src/main.rs

@@ -1,12 +1,9 @@
 use futures::TryStreamExt;
-use geozero;
-use geozero::mvt::{self, tile, Message, Tile};
-use geozero::{ProcessToJson, ToJson};
+
+use geozero::mvt::{Message, Tile};
+
 use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
-use sqlx::{
-    query, query_as, query_as_unchecked, ConnectOptions, Executor, FromRow, Statement,
-};
-use std::fmt::Pointer;
+use sqlx::{query_as, ConnectOptions, Executor, FromRow};
 
 // #[derive(Debug, FromRow)]
 // struct MetadataRow {
@@ -36,7 +33,7 @@ fn mvt_dev() {
     println!("bytes: {:?}", bytes.len());
     // let buf = bytes.as_slice();
 
-    let mut cursor = std::io::Cursor::new(bytes.as_slice());
+    let cursor = std::io::Cursor::new(bytes.as_slice());
 
     let mt = Tile::decode(cursor).unwrap();
 
@@ -90,7 +87,7 @@ async fn sqlxing() {
     let copts = SqliteConnectOptions::new()
         .filename(file)
         .create_if_missing(true);
-    let mut c = copts.connect().await.unwrap();
+    let _c = copts.connect().await.unwrap();
 
     let pool = SqlitePoolOptions::new()
         .max_connections(5)
@@ -101,9 +98,8 @@ async fn sqlxing() {
     // timing
     // start
     let start = std::time::Instant::now();
-    // let tthingydickr = query_as::<_, MetadataRow>("SELECT * FROM tiles");
     let mut r = query_as::<_, MetadataRow>("SELECT * FROM tiles").fetch(&pool);
-    while let Some(row) = r.try_next().await.unwrap() {
+    while let Some(_row) = r.try_next().await.unwrap() {
         // println!("row: {:?}", row);
     }