Add MODNet model (#11)

* Add MODNet for portrait matting

* Minor fixes

* Move assets to home directory

* Add colormap

* ci

* Update README.md

.github/workflows/rust-ci.yml

@@ -2,7 +2,7 @@ name: Rust
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   pull_request:
     branches: [ "main" ]
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usls"
-version = "0.0.2"
+version = "0.0.3"
 edition = "2021"
 description = "A Rust library integrated with ONNXRuntime, providing a collection of ML models."
 repository = "https://github.com/jamjamjon/usls"

README.md

@@ -1,18 +1,22 @@
 # usls
 
-A Rust library integrated with **ONNXRuntime**, providing a collection of **Computer Vison** and **Vision-Language** models including [YOLOv5](https://github.com/ultralytics/yolov5), [YOLOv8](https://github.com/ultralytics/ultralytics), [YOLOv9](https://github.com/WongKinYiu/yolov9), [RTDETR](https://arxiv.org/abs/2304.08069), [CLIP](https://github.com/openai/CLIP), [DINOv2](https://github.com/facebookresearch/dinov2), [FastSAM](https://github.com/CASIA-IVA-Lab/FastSAM), [YOLO-World](https://github.com/AILab-CVC/YOLO-World), [BLIP](https://arxiv.org/abs/2201.12086), [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR) , [Depth-Anything](https://github.com/LiheYoung/Depth-Anything) and others.
+A Rust library integrated with **ONNXRuntime**, providing a collection of **Computer Vison** and **Vision-Language** models including [YOLOv5](https://github.com/ultralytics/yolov5), [YOLOv8](https://github.com/ultralytics/ultralytics), [YOLOv9](https://github.com/WongKinYiu/yolov9), [RTDETR](https://arxiv.org/abs/2304.08069), [CLIP](https://github.com/openai/CLIP), [DINOv2](https://github.com/facebookresearch/dinov2), [FastSAM](https://github.com/CASIA-IVA-Lab/FastSAM), [YOLO-World](https://github.com/AILab-CVC/YOLO-World), [BLIP](https://arxiv.org/abs/2201.12086), [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR), [Depth-Anything](https://github.com/LiheYoung/Depth-Anything), [MODNet](https://github.com/ZHKKKe/MODNet) and others.
 
 ## Recently Updated
 
+|        Portrait Matting         |
+| :----------------------------: |
+|<img src='examples/modnet/demo.png'   width="800px">|
+
+
 |        Depth-Anything         |
 | :----------------------------: |
 |<img src='examples/depth-anything/demo.png'   width="800px">|
 
 
-
 |        YOLOP-v2          |             Face-Parsing             |               Text-Detection           |  
 | :----------------------------: | :------------------------------: |  :------------------------------: |
-|<img src='examples/yolop/demo.png'  height="240px">| <img src='examples/face-parsing/demo.png'  height="240px"> | <img src='examples/db/demo.png'  height="240px"> |
+|<img src='examples/yolop/demo.png'  height="180px">| <img src='examples/face-parsing/demo.png'  height="180px"> | <img src='examples/db/demo.png'  height="180px"> |
 
 
 |        YOLOv8-Obb         |
@@ -23,8 +27,6 @@ A Rust library integrated with **ONNXRuntime**, providing a collection of **Comp
 
 
 
-
-
 ## Supported Models
 
 |                               Model                               |         Task / Type         |         Example         | CUDA<br />f32 | CUDA<br />f16 |     TensorRT<br />f32     |     TensorRT<br />f16     |
@@ -48,6 +50,7 @@ A Rust library integrated with **ONNXRuntime**, providing a collection of **Comp
 |    [YOLOv5-classification](https://github.com/ultralytics/yolov5)    |      Object Detection      |   [demo](examples/yolov5)   |      ✅      |      ✅      |             ✅             |            ✅            |
 |     [YOLOv5-segmentation](https://github.com/ultralytics/yolov5)     |    Instance Segmentation    |   [demo](examples/yolov5)   |      ✅      |      ✅      |             ✅             |            ✅            |
 |     [Depth-Anything](https://github.com/LiheYoung/Depth-Anything)     |    Monocular Depth Estimation    |   [demo](examples/depth-anything)   |      ✅      |      ✅      |             ❌             |            ❌            |
+|     [MODNet](https://github.com/ZHKKKe/MODNet)     |    Image Matting    |   [demo](examples/modnet)   |      ✅      |      ✅      |             ✅             |            ✅            |
 
 ## Solution Models
 

examples/blip/main.rs

@@ -3,14 +3,14 @@ use usls::{models::Blip, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // visual
     let options_visual = Options::default()
-        .with_model("../models/blip-visual-base.onnx")
+        .with_model("blip-visual-base.onnx")?
         .with_i00((1, 1, 4).into())
         .with_profile(false);
 
     // textual
     let options_textual = Options::default()
-        .with_model("../models/blip-textual-base.onnx")
-        .with_tokenizer("tokenizer-blip.json")
+        .with_model("blip-textual-base.onnx")?
+        .with_tokenizer("tokenizer-blip.json")?
         .with_i00((1, 1, 4).into()) // input_id: batch
         .with_i01((1, 1, 4).into()) // input_id: seq_len
         .with_i10((1, 1, 4).into()) // attention_mask: batch

examples/clip/main.rs

@@ -3,14 +3,14 @@ use usls::{models::Clip, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // visual
     let options_visual = Options::default()
-        .with_model("../models/clip-b32-visual-dyn.onnx")
+        .with_model("clip-b32-visual-dyn.onnx")?
         .with_i00((1, 1, 4).into())
         .with_profile(false);
 
     // textual
     let options_textual = Options::default()
-        .with_model("../models/clip-b32-textual-dyn.onnx")
-        .with_tokenizer("tokenizer-clip.json")
+        .with_model("clip-b32-textual-dyn.onnx")?
+        .with_tokenizer("tokenizer-clip.json")?
         .with_i00((1, 1, 4).into())
         .with_profile(false);
 

examples/db/main.rs

@@ -10,7 +10,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .with_min_width(5.0)
         .with_min_height(12.0)
         // .with_trt(0)
-        .with_model("../models/ppocr-v4-db-dyn.onnx");
+        .with_model("ppocr-v4-db-dyn.onnx")?;
 
     let mut model = DB::new(&options)?;
 
@@ -26,8 +26,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // annotate
     let annotator = Annotator::default()
         .without_bboxes(true)
-        .with_masks_alpha(60)
-        .with_polygon_color([255, 105, 180, 255])
+        .with_polygons_alpha(60)
+        .with_contours_color([255, 105, 180, 255])
         .without_mbrs(true)
         .with_saveout("DB");
     annotator.annotate(&x, &y);

examples/depth-anything/main.rs

@@ -3,7 +3,7 @@ use usls::{models::DepthAnything, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // visual
     let options = Options::default()
-        .with_model("../models/depth-anything-s-dyn.onnx")
+        .with_model("depth-anything-s-dyn.onnx")?
         .with_i00((1, 1, 8).into())
         .with_i02((384, 512, 1024).into())
         .with_i03((384, 512, 1024).into());
@@ -16,7 +16,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let y = model.run(&x)?;
 
     // annotate
-    let annotator = Annotator::default().with_saveout("Depth-Anything");
+    let annotator = Annotator::default()
+        .with_colormap("Turbo")
+        .with_saveout("Depth-Anything");
     annotator.annotate(&x, &y);
 
     Ok(())

examples/dinov2/main.rs

@@ -3,7 +3,7 @@ use usls::{models::Dinov2, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/dinov2-s14-dyn-f16.onnx")
+        .with_model("dinov2-s14-dyn-f16.onnx")?
         .with_i00((1, 1, 1).into())
         .with_i02((224, 224, 224).into())
         .with_i03((224, 224, 224).into());

examples/face-parsing/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/face-parsing-dyn.onnx")
+        .with_model("face-parsing-dyn.onnx")?
         .with_i00((1, 1, 4).into())
         .with_i02((416, 640, 800).into())
         .with_i03((416, 640, 800).into())
@@ -23,8 +23,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .without_bboxes(true)
         .without_bboxes_conf(true)
         .without_bboxes_name(true)
-        .without_polygons(false)
-        .with_masks_name(false)
+        .without_contours(false)
+        .with_polygons_name(false)
         .with_saveout("Face-Parsing");
     annotator.annotate(&x, &y);
 

examples/fastsam/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/FastSAM-s-dyn-f16.onnx")
+        .with_model("FastSAM-s-dyn-f16.onnx")?
         .with_i00((1, 1, 4).into())
         .with_i02((416, 640, 800).into())
         .with_i03((416, 640, 800).into())

examples/modnet/README.md

@@ -0,0 +1,15 @@
+## Quick Start
+
+```shell
+cargo run -r --example modnet
+```
+
+## ONNX Model
+
+- [modnet-dyn](https://github.com/jamjamjon/assets/releases/download/v0.0.1/modnet-dyn.onnx)  
+
+
+
+## Results
+
+![](./demo.png)

examples/modnet/main.rs

@@ -0,0 +1,23 @@
+use usls::{models::MODNet, Annotator, DataLoader, Options};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // build model
+    let options = Options::default()
+        .with_model("modnet-dyn.onnx")?
+        .with_i00((1, 1, 4).into())
+        .with_i02((416, 512, 800).into())
+        .with_i03((416, 512, 800).into());
+    let model = MODNet::new(&options)?;
+
+    // load image
+    let x = vec![DataLoader::try_read("./assets/portrait.jpg")?];
+
+    // run
+    let y = model.run(&x)?;
+
+    // annotate
+    let annotator = Annotator::default().with_saveout("MODNet");
+    annotator.annotate(&x, &y);
+
+    Ok(())
+}

examples/rtdetr/main.rs

@@ -3,7 +3,7 @@ use usls::{coco, models::RTDETR, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/rtdetr-l-f16.onnx")
+        .with_model("rtdetr-l-f16.onnx")?
         .with_confs(&[0.4, 0.15]) // person: 0.4, others: 0.15
         .with_names(&coco::NAMES_80);
     let mut model = RTDETR::new(&options)?;

examples/rtmo/main.rs

@@ -3,7 +3,7 @@ use usls::{coco, models::RTMO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../rtmo-s-dyn.onnx")
+        .with_model("rtmo-s-dyn.onnx")?
         .with_i00((1, 1, 8).into())
         .with_nk(17)
         .with_confs(&[0.3])

examples/svtr/main.rs

@@ -6,8 +6,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .with_i00((1, 2, 8).into())
         .with_i03((320, 960, 1600).into())
         .with_confs(&[0.2])
-        .with_vocab("../ppocr_rec_vocab.txt")
-        .with_model("../models/ppocr-v4-svtr-ch-dyn.onnx");
+        .with_vocab("ppocr_rec_vocab.txt")?
+        .with_model("ppocr-v4-svtr-ch-dyn.onnx")?;
     let mut model = SVTR::new(&options)?;
 
     // load images

examples/yolo-world/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/yolov8s-world-v2-shoes.onnx")
+        .with_model("yolov8s-world-v2-shoes.onnx")?
         .with_i00((1, 1, 4).into())
         .with_i02((416, 640, 800).into())
         .with_i03((416, 640, 800).into())

examples/yolop/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLOPv2, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/yolopv2-dyn-480x800.onnx")
+        .with_model("yolopv2-dyn-480x800.onnx")?
         .with_i00((1, 1, 8).into())
         .with_confs(&[0.3]);
     let mut model = YOLOPv2::new(&options)?;
@@ -16,7 +16,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // annotate
     let annotator = Annotator::default()
-        .with_masks_name(true)
+        .with_polygons_name(true)
         .with_saveout("YOLOPv2");
     annotator.annotate(&x, &y);
 

examples/yolov5/main.rs

@@ -9,7 +9,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .with_conf_independent(true)
         .with_anchors_first(true)
         .with_yolo_task(YOLOTask::Segment)
-        .with_model("../models/yolov5s-seg.onnx")
+        .with_model("yolov5s-seg.onnx")?
         .with_trt(0)
         .with_fp16(true)
         .with_i00((1, 1, 4).into())

examples/yolov8-face/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/yolov8n-face-dyn-f16.onnx")
+        .with_model("yolov8n-face-dyn-f16.onnx")?
         .with_i00((1, 1, 4).into())
         .with_i02((416, 640, 800).into())
         .with_i03((416, 640, 800).into())

examples/yolov8-falldown/main.rs

@@ -2,7 +2,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
-    let options = Options::default().with_model("../models/yolov8-falldown-f16.onnx");
+    let options = Options::default().with_model("yolov8-falldown-f16.onnx")?;
     let mut model = YOLO::new(&options)?;
 
     // load image

examples/yolov8-head/main.rs

@@ -2,7 +2,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
-    let options = Options::default().with_model("../models/yolov8-head-f16.onnx");
+    let options = Options::default().with_model("yolov8-head-f16.onnx")?;
     let mut model = YOLO::new(&options)?;
 
     // load image

examples/yolov8-trash/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // 1.build model
     let options = Options::default()
-        .with_model("../models/yolov8-plastic-bag-f16.onnx")
+        .with_model("yolov8-plastic-bag-f16.onnx")?
         .with_names(&["trash"]);
     let mut model = YOLO::new(&options)?;
 

examples/yolov8/main.rs

@@ -3,23 +3,20 @@ use usls::{coco, models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        // .with_model("../models/yolov8m.onnx")
-        // .with_model("../models/yolov8m-dyn-f16.onnx")
-        // .with_model("../models/yolov8m-pose-dyn-f16.onnx")
-        // .with_model("../models/yolov8m-seg-dyn-f16.onnx")
-        .with_model("../models/yolov8s-cls.onnx")
-        // .with_model("../models/yolov8s-obb.onnx")
+        .with_model("yolov8m-dyn.onnx")?
+        // .with_model("yolov8m-pose-dyn.onnx")?
+        // .with_model("yolov8m-cls-dyn.onnx")?
+        // .with_model("yolov8m-seg-dyn.onnx")?
+        // .with_model("yolov8m-obb-dyn.onnx")?
+        // .with_model("yolov8m-oiv7-dyn.onnx")?
         // .with_trt(0)
         // .with_fp16(true)
         .with_i00((1, 1, 4).into())
-        .with_i02((224, 1024, 1024).into())
-        .with_i03((224, 1024, 1024).into())
-        // .with_i02((224, 640, 800).into())
-        // .with_i03((224, 640, 800).into())
+        .with_i02((224, 640, 800).into())
+        .with_i03((224, 640, 800).into())
         .with_confs(&[0.4, 0.15]) // person: 0.4, others: 0.15
         .with_names2(&coco::KEYPOINTS_NAMES_17)
-        .with_profile(true)
-        .with_dry_run(10);
+        .with_profile(false);
     let mut model = YOLO::new(&options)?;
 
     // build dataloader
@@ -30,41 +27,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // build annotate
     let annotator = Annotator::default()
-        // .with_probs_topk(10)
-        // // bboxes
-        // .without_bboxes(false)
-        // .without_bboxes_conf(false)
-        // .without_bboxes_name(false)
-        // .without_bboxes_text_bg(false)
-        // .with_bboxes_text_color([255, 255, 255, 255])
-        // .with_bboxes_text_bg_alpha(255)
-        // // keypoints
-        // .without_keypoints(false)
-        // .with_keypoints_palette(&COCO_KEYPOINT_COLORS_17)
         .with_skeletons(&coco::SKELETONS_16)
-        // .with_keypoints_name(false)
-        // .with_keypoints_conf(false)
-        // .without_keypoints_text_bg(false)
-        // .with_keypoints_text_color([255, 255, 255, 255])
-        // .with_keypoints_text_bg_alpha(255)
-        // .with_keypoints_radius(4)
-        // // masks
-        // .without_masks(false)
-        // .with_masks_alpha(190)
-        // .without_polygons(false)
-        // // .with_polygon_color([0, 255, 255, 255])
-        // .with_masks_conf(false)
-        // .with_masks_name(true)
-        // .with_masks_text_bg(true)
-        // .with_masks_text_color([255, 255, 255, 255])
-        // .with_masks_text_bg_alpha(10)
-        // // mbrs
-        // .without_mbrs(false)
-        // .without_mbrs_conf(false)
-        // .without_mbrs_name(false)
-        // .without_mbrs_text_bg(false)
-        // .with_mbrs_text_color([255, 255, 255, 255])
-        // .with_mbrs_text_bg_alpha(70)
         .with_saveout("YOLOv8");
 
     // run & annotate

examples/yolov9/main.rs

@@ -3,7 +3,7 @@ use usls::{models::YOLO, Annotator, DataLoader, Options};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // build model
     let options = Options::default()
-        .with_model("../models/yolov9-c-dyn-f16.onnx")
+        .with_model("yolov9-c-dyn-f16.onnx")?
         .with_i00((1, 1, 4).into())
         .with_i02((416, 640, 800).into())
         .with_i03((416, 640, 800).into())