Merge commit '90b8d3045fe7b483ebea0b7bf0d31a4368bc2e49' into adapt-to…

…-voicevox-core
VOICEVOX · Feb 11, 2024 · 8e37579 · 8e37579
2 parents 5dd2fcb + 90b8d30
commit 8e37579
Show file tree

Hide file tree

Showing 26 changed files with 715 additions and 160 deletions.
diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
@@ -73,6 +73,7 @@ jobs:
         run: |
           cargo tarpaulin -p ort --features fetch-models --verbose --timeout 120 --out xml
       - name: Upload to codecov.io
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
diff --git a/.gitignore b/.gitignore
@@ -191,3 +191,6 @@ WixTools/
 
 # IDEA
 .idea
+
+# Glassbench results
+/glassbench*.db
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,19 +3,21 @@ members = [
 	'ort-sys',
 	'examples/gpt2',
 	'examples/model-info',
-	'examples/yolov8'
+	'examples/yolov8',
+	'examples/modnet'
 ]
 default-members = [
 	'.',
 	'examples/gpt2',
 	'examples/model-info',
-	'examples/yolov8'
+	'examples/yolov8',
+	'examples/modnet'
 ]
 
 [package]
 name = "voicevox-ort"
-description = "A safe Rust wrapper for ONNX Runtime 1.16 - Optimize and Accelerate Machine Learning Inferencing"
-version = "2.0.0-alpha.4"
+description = "A safe Rust wrapper for ONNX Runtime 1.17 - Optimize and Accelerate Machine Learning Inferencing"
+version = "2.0.0-rc.0"
 edition = "2021"
 rust-version = "1.70"
 license = "MIT OR Apache-2.0"
@@ -28,7 +30,7 @@ authors = [
 	"pyke.io <[email protected]>",
 	"Nicolas Bigaouette <[email protected]>"
 ]
-include = [ "src/", "examples/", "tests/", "LICENSE-APACHE", "LICENSE-MIT", "README.md" ]
+include = [ "src/", "LICENSE-APACHE", "LICENSE-MIT", "README.md" ]
 
 [profile.release]
 opt-level = 3
@@ -77,6 +79,7 @@ ndarray = { version = "0.15", optional = true }
 thiserror = "1.0"
 voicevox-ort-sys = { version = "2.0.0-alpha.4", path = "ort-sys" }
 libloading = { version = "0.8", optional = true }
+compact_str = "0.7"
 
 ureq = { version = "2.1", optional = true, default-features = false, features = [ "tls" ] }
 tracing = "0.1"
@@ -95,3 +98,8 @@ ureq = "2.1"
 image = "0.24"
 test-log = { version = "0.2", default-features = false, features = [ "trace" ] }
 tracing-subscriber = { version = "0.3", default-features = false, features = [ "env-filter", "fmt" ] }
+glassbench = "0.4"
+
+[[bench]]
+name = "squeezenet"
+harness = false
diff --git a/README.md b/README.md
@@ -3,10 +3,10 @@
     <hr />
     <a href="https://app.codecov.io/gh/pykeio/ort" target="_blank"><img alt="Coverage Results" src="https://img.shields.io/codecov/c/gh/pykeio/ort?style=for-the-badge"></a> <a href="https://crates.io/crates/ort" target="_blank"><img alt="Crates.io" src="https://img.shields.io/crates/d/ort?style=for-the-badge"></a> <a href="https://opencollective.com/pyke-osai" target="_blank"><img alt="Open Collective backers and sponsors" src="https://img.shields.io/opencollective/all/pyke-osai?style=for-the-badge&label=sponsors"></a>
     <br />
-    <a href="https://crates.io/crates/ort" target="_blank"><img alt="Crates.io" src="https://img.shields.io/crates/v/ort?style=for-the-badge&label=ort&logo=rust"></a> <img alt="ONNX Runtime" src="https://img.shields.io/badge/onnxruntime-v1.16.3-blue?style=for-the-badge&logo=cplusplus">
+    <a href="https://crates.io/crates/ort" target="_blank"><img alt="Crates.io" src="https://img.shields.io/crates/v/ort?style=for-the-badge&label=ort&logo=rust"></a> <img alt="ONNX Runtime" src="https://img.shields.io/badge/onnxruntime-v1.17.0-blue?style=for-the-badge&logo=cplusplus">
 </div>
 
-`ort` is an (unofficial) [ONNX Runtime](https://onnxruntime.ai/) 1.16 wrapper for Rust based on the now inactive [`onnxruntime-rs`](https://github.com/nbigaouette/onnxruntime-rs). ONNX Runtime accelerates ML inference on both CPU & GPU.
+`ort` is an (unofficial) [ONNX Runtime](https://onnxruntime.ai/) 1.17 wrapper for Rust based on the now inactive [`onnxruntime-rs`](https://github.com/nbigaouette/onnxruntime-rs). ONNX Runtime accelerates ML inference on both CPU & GPU.
 
 ## 📖 Documentation
 - [Guide](https://ort.pyke.io/)

diff --git a/benches/squeezenet.rs b/benches/squeezenet.rs
@@ -0,0 +1,60 @@
+use std::{path::Path, sync::Arc};
+
+use glassbench::{pretend_used, Bench};
+use image::{imageops::FilterType, ImageBuffer, Pixel, Rgb};
+use ndarray::{s, Array4};
+use ort::{GraphOptimizationLevel, Session};
+
+fn load_squeezenet_data() -> ort::Result<(Session, Array4<f32>)> {
+	const IMAGE_TO_LOAD: &str = "mushroom.png";
+
+	ort::init().with_name("integration_test").commit()?;
+
+	let session = Session::builder()?
+		.with_optimization_level(GraphOptimizationLevel::Level1)?
+		.with_intra_threads(1)?
+		.with_model_downloaded("https://parcel.pyke.io/v2/cdn/assetdelivery/ortrsv2/ex_models/squeezenet.onnx")
+		.expect("Could not download model from file");
+
+	let input0_shape: &Vec<i64> = session.inputs[0].input_type.tensor_dimensions().expect("input0 to be a tensor type");
+
+	let image_buffer: ImageBuffer<Rgb<u8>, Vec<u8>> = image::open(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests").join("data").join(IMAGE_TO_LOAD))
+		.unwrap()
+		.resize(input0_shape[2] as u32, input0_shape[3] as u32, FilterType::Nearest)
+		.to_rgb8();
+
+	let mut array = ndarray::Array::from_shape_fn((1, 3, 224, 224), |(_, c, j, i)| {
+		let pixel = image_buffer.get_pixel(i as u32, j as u32);
+		let channels = pixel.channels();
+		(channels[c] as f32) / 255.0
+	});
+
+	let mean = [0.485, 0.456, 0.406];
+	let std = [0.229, 0.224, 0.225];
+	for c in 0..3 {
+		let mut channel_array = array.slice_mut(s![0, c, .., ..]);
+		channel_array -= mean[c];
+		channel_array /= std[c];
+	}
+
+	Ok((session, array))
+}
+
+fn bench_squeezenet(bench: &mut Bench) {
+	let (session, data) = load_squeezenet_data().unwrap();
+	bench.task("ArrayView", |task| {
+		task.iter(|| {
+			pretend_used(session.run(ort::inputs![data.view()].unwrap()).unwrap());
+		})
+	});
+
+	let raw = Arc::new(data.as_standard_layout().as_slice().unwrap().to_owned().into_boxed_slice());
+	let shape: Vec<i64> = data.shape().iter().map(|c| *c as _).collect();
+	bench.task("Raw data", |task| {
+		task.iter(|| {
+			pretend_used(session.run(ort::inputs![(shape.clone(), Arc::clone(&raw))].unwrap()).unwrap());
+		})
+	});
+}
+
+glassbench::glassbench!("SqueezeNet", bench_squeezenet,);
diff --git a/codecov.yml b/codecov.yml
@@ -1,3 +1,2 @@
 ignore:
-  - src/download/**/*.rs
-  - src/download.rs
+  - "src/execution_providers"
diff --git a/docs/introduction.mdx b/docs/introduction.mdx
@@ -3,49 +3,58 @@ title: Introduction
 ---
 
 <p style={{ fontSize: '1.5rem', textAlign: 'center' }}>
-    `ort` is an open-source Rust binding for [ONNX Runtime](https://github.com/microsoft/onnxruntime).
+    `ort` is an open-source Rust binding for [ONNX Runtime](https://onnxruntime.ai/).
 </p>
 
-`ort` makes it easy to deploy your machine learning models to production via ONNX Runtime, a [hardware-accelerated](/perf/execution-providers) inference engine. With `ort` + ONNX Runtime, you can run almost any ML model (including ResNet, YOLOv8, BERT, LLaMA), often much faster than PyTorch, and with the added bonus of Rust's efficiency.
+`ort` makes it easy to deploy your machine learning models to production via [ONNX Runtime](https://onnxruntime.ai/), a hardware-accelerated inference engine. With `ort` + ONNX Runtime, you can run almost any ML model (including ResNet, YOLOv8, BERT, LLaMA) on almost any hardware, often far faster than PyTorch, and with the added bonus of Rust's efficiency.
+
+<Warning>
+    These docs are for the latest alpha version of `ort`, `2.0.0-rc.0`. This version is production-ready (just not API stable) and we recommend new & existing projects use it.
+</Warning>
 
 # Why `ort`?
 There are a few other ONNX Runtime crates out there, so why use `ort`?
 
 For one, `ort` simply supports more features:
 
-| Feature comparison     | **📕 ort** | **📗 [ors](https://github.com/HaoboGu/ors)** | **🪟 [onnxruntime-rs](https://github.com/microsoft/onnxruntime/tree/main/rust)** |
-|------------------------|-----------|-----------|----------------------|
-| Upstream version       | **v1.16.3** | v1.12.0 | v1.8               |
-| `dlopen()`?            | ✅         | ✅         | ❌                    |
-| Execution providers?   | ✅         | ❌         | ❌                    |
-| IOBinding?             | ✅         | ❌         | ❌                    |
-| String tensors?        | ✅         | ❌         | ⚠️ input only         |
-| Multiple output types? | ✅         | ✅         | ❌                    |
-| Multiple input types?  | ✅         | ✅         | ❌                    |
-| In-memory session?     | ✅         | ✅         | ✅                    |
-| WebAssembly?           | ✅         | ❌         | ❌                    |
+| Feature comparison        | **📕 ort** | **📗 [ors](https://github.com/HaoboGu/ors)** | **🪟 [onnxruntime-rs](https://github.com/microsoft/onnxruntime/tree/main/rust)** |
+|---------------------------|-----------|-----------|----------------------|
+| Upstream version          | **v1.17.0** | v1.12.0 | v1.8               |
+| `dlopen()`?               | ✅         | ✅         | ❌                    |
+| Execution providers?      | ✅         | ❌         | ❌                    |
+| I/O Binding?              | ✅         | ❌         | ❌                    |
+| String tensors?           | ✅         | ❌         | ⚠️ input only         |
+| Multiple output types?    | ✅         | ✅         | ❌                    |
+| Multiple input types?     | ✅         | ✅         | ❌                    |
+| In-memory session?        | ✅         | ✅         | ✅                    |
+| WebAssembly?              | ✅         | ❌         | ❌                    |
+| Provides static binaries? | ✅         | ❌         | ❌                    |
+| Sequence & map types?     | ✅         | ❌         | ❌                    |
 
 Users of `ort` appreciate its ease of use and ergonomic API. `ort` is also battle tested in some pretty serious production scenarios.
 - [**Twitter**](https://twitter.com/) uses `ort` in part of their recommendations system, serving hundreds of millions of requests a day.
 - [**Bloop**](https://bloop.ai/)'s semantic code search feature is powered by `ort`.
+- [**SurrealDB**](https://surrealdb.com/) uses `ort` in their [`surrealml`](https://github.com/surrealdb/surrealml) package.
 - [**Numerical Elixir**](https://github.com/elixir-nx) uses `ort` to create ONNX Runtime bindings for the Elixir language.
-- [**`rust-bert`**](https://github.com/guillaume-be/rust-bert) implements many ready-to-use NLP pipelines in Rust a la Hugging Face Transformers, with an optional `ort` backend.
+- [**`rust-bert`**](https://github.com/guillaume-be/rust-bert) implements many ready-to-use NLP pipelines in Rust à la Hugging Face Transformers with both [`tch`](https://crates.io/crates/tch) & `ort` backends.
 - [**`edge-transformers`**](https://github.com/npc-engine/edge-transformers) also implements Hugging Face Transformers pipelines in Rust using `ort`.
-- We use `ort` in nearly all of our ML projects, including [VITRI](https://vitri.pyke.io/) 😊
 
 # Getting started
 <Steps>
     <Step title="Add ort to your Cargo.toml">
-        If you have a [supported platform](/setup/platforms), installing `ort` couldn't be any simpler!
+        If you have a [supported platform](/setup/platforms) (and you probably do), installing `ort` couldn't be any simpler! Just add it to your Cargo dependencies:
         ```toml
         [dependencies]
-        ort = "2.0"
+        ort = "2.0.0-alpha.4"
         ```
     </Step>
     <Step title="Convert your model">
-        Your model will need to be converted to the [ONNX](https://onnx.ai/) format before you can use it; here's how to do that:
-        - The awesome folks at Hugging Face have [a guide](https://huggingface.co/docs/transformers/serialization) to export Transformers models to ONNX with 🤗 Optimum.
-        - For other PyTorch models, see the [`torch.onnx` module docs](https://pytorch.org/docs/stable/onnx.html).
+        Your model will need to be converted to the [ONNX](https://onnx.ai/) format before you can use it.
+        - The awesome folks at Hugging Face have [a guide](https://huggingface.co/docs/transformers/serialization) to export 🤗 Transformers models to ONNX with 🤗 Optimum.
+        - For other PyTorch models: [`torch.onnx`](https://pytorch.org/docs/stable/onnx.html)
+        - For `scikit-learn`: [`sklearn-onnx`](https://onnx.ai/sklearn-onnx/)
+        - For TensorFlow, Keras, TFlite, TensorFlow.js: [`tf2onnx`](https://github.com/onnx/tensorflow-onnx)
+        - For PaddlePaddle: [`Paddle2ONNX`](https://github.com/PaddlePaddle/Paddle2ONNX)
     </Step>
     <Step title="Load your model">
         Once you've got a model, load it via `ort` by creating a [`Session`](/fundamentals/session):
@@ -64,6 +73,8 @@ Users of `ort` appreciate its ease of use and ergonomic API. `ort` is also battl
 
         ```rust
         let outputs = model.run(ort::inputs!["image" => image]?)?;
+
+        // Postprocessing
         let output = outputs["output0"]
             .extract_tensor::<f32>()
             .unwrap()
@@ -74,7 +85,7 @@ Users of `ort` appreciate its ease of use and ergonomic API. `ort` is also battl
         ...
         ```
 
-        <Note>There are some [more useful examples](https://github.com/pykeio/ort/tree/main/examples) in our repo!</Note>
+        <Note>There are some more useful examples [in the `ort` repo](https://github.com/pykeio/ort/tree/main/examples)!</Note>
     </Step>
 </Steps>
 

diff --git a/docs/migrating/v2.mdx b/docs/migrating/v2.mdx
@@ -14,7 +14,7 @@ ort::init()
     .commit()?;
 ```
 
-`commit()` must be called before any sessions are created to take effect. Otherwise, the environment will be set to default and cannot be modified afterwards.
+`commit()` must be called before any sessions are created to take effect. Otherwise, a default environment will be created. The global environment can be updated afterward by calling `commit()` on another `EnvironmentBuilder`, however you'll need to recreate sessions after comitting the new environment in order for them to use it.
 
 ## Session creation
 `SessionBuilder::new(&environment)` has been soft-replaced with `Session::builder()`:
@@ -144,6 +144,14 @@ The `ort::sys` module has been split out into [its own `ort-sys` crate](https://
 ### `ndarray` is now optional
 The dependency on `ndarray` is now declared optional. If you use `ort` with `default-features = false`, you'll need to add the `ndarray` feature.
 
+## Model Zoo structs have been removed
+ONNX pushed a new Model Zoo structure that adds hundreds of different models. This is impractical to maintain, so the built-in structs have been removed.
+
+You can still use `Session::with_model_downloaded`, it just now takes a URL string instead of a struct.
+
+## Changes to logging
+Environment-level logging configuration (i.e. `EnvironmentBuilder::with_log_level`) has been removed because it could cause unnecessary confusion with our `tracing` integration.
+
 ## The Flattening
 All modules except `download` are no longer public. Exports have been flattened to the crate root, so i.e. `ort::session::Session` becomes `ort::Session`.
 
@@ -153,3 +161,4 @@ The following types have been renamed with no other changes.
 - `OrtOwnedTensor` -> `Tensor`
 - `OrtResult`, `OrtError` -> `ort::Result`, `ort::Error`
 - `TensorDataToType` -> `ExtractTensorData`
+- `TensorElementDataType`, `IntoTensorElementDataType` -> `TensorElementType`, `IntoTensorElementType`
diff --git a/docs/migrating/version-mapping.mdx b/docs/migrating/version-mapping.mdx
@@ -6,7 +6,7 @@ description: Information about `ort`'s versioning and relation to ONNX Runtime v
 ## A note on SemVer
 `ort` versions pre-2.0 were not SemVer compatible. From v2.0 onwards, breaking API changes are accompanied by a **major version update**.
 
-Updates to the version of ONNX Runtime used by `ort` may occur on **minor** version updates, i.e. 2.0 ships with ONNX Runtime 1.16.2, but 2.1 may ship with 1.17.0. ONNX Runtime is generally forward compatible, but in case you require a specific version of ONNX Runtime, you should pin the minor version in your `Cargo.toml` using a [tilde requirement](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#tilde-requirements):
+Updates to the version of ONNX Runtime used by `ort` may occur on **minor** version updates, i.e. 2.0 ships with ONNX Runtime 1.17.0, but 2.1 may ship with 1.18.0. ONNX Runtime is generally forward compatible, but in case you require a specific version of ONNX Runtime, you should pin the minor version in your `Cargo.toml` using a [tilde requirement](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#tilde-requirements):
 ```toml
 [dependencies]
 ort = { version = "~2.0", ... }
@@ -16,12 +16,10 @@ ort = { version = "~2.0", ... }
 
 | **ort**  | **ONNX Runtime** |
 | -------- | ----------------:|
-| v2.0.0+ | v1.16.2 |
+| v2.0.0+ | v1.17.0 |
 | v1.16.0-v1.16.2 | v1.16.0 |
 | v1.15.0-v1.15.5 | v1.15.1 |
 | v1.14.2-v1.14.8 | v1.14.1 |
 | v1.14.0-v1.14.1 | v1.14.0 |
 | v1.13.1-v1.13.3 | v1.13.1 |
 | v1.13.0  | v1.12.1 |
-
-If you need support for an old (&lt;1.15) version of `ort`, or need an even older version of ONNX Runtime, [contact us](mailto:[email protected]).
diff --git a/docs/setup/linking.mdx b/docs/setup/linking.mdx
@@ -32,7 +32,7 @@ To use `load-dynamic`:
         ort = { version = "2", features = [ "load-dynamic" ] }
         ```
     </Step>
-    <Step title="Point ort to the dylib with ORT_DYLIB_PATH">
+    <Step title="Point ort to the dylib">
         <Tabs>
             <Tab title="Via shell">
                 ```shell
@@ -41,22 +41,15 @@ To use `load-dynamic`:
             </Tab>
             <Tab title="Programmatically">
                 ```rust
-                use std::env;
-
                 fn main() -> anyhow::Result<()> {
-                    // Find our downloaded ONNX Runtime dylibs and set the environment variable for ort
-                    env::set_var("ORT_DYLIB_PATH", crate::internal::find_onnxruntime_dylib()?);
+                    // Find our custom ONNX Runtime dylibs and initialize `ort` with it.
+                    let dylib_path = crate::internal::find_onnxruntime_dylib()?; // /etc/.../libonnxruntime.so
 
-                    // IMPORTANT: You must set the environment variable **before** you use `ort`!!!
-                    ort::init().commit()?;
+                    ort::init_from(dylib_path).commit()?;
 
                     Ok(())
                 }
                 ```
-
-                <Warning>
-                    You MUST set the environment variable **before** you use any `ort` APIs!
-                </Warning>
             </Tab>
         </Tabs>
     </Step>

diff --git a/docs/setup/platforms.mdx b/docs/setup/platforms.mdx
@@ -12,13 +12,18 @@ Here are the supported platforms and binary availability status, as of v2.0.
 
 | Platform | x86 | x86-64 | ARMv7 | ARM64 | WASM32 |
 |:-------- |:------- |:------ |:------ |:------ |:------ |
-| **Windows** | ⭕ | 🟢 | ⭕ | 🔷 | ❌ |
-| **Linux** | ⭕ | 🟢 | ⭕ | 🔷 | ❌ |
-| **macOS** | ❌ | 🔷 | ❌ | 🔷 | ❌ |
+| **Windows** | ⭕ | 🟢<sup>\*</sup> | ⭕ | 🔷<sup>\*</sup> | ❌ |
+| **Linux** | ⭕ | 🟢† | ⭕ | 🔷‡ | ❌ |
+| **macOS** | ❌ | 🔷§ | ❌ | 🔷§ | ❌ |
 | **iOS** | ❌ | ❌ | ❌ | ⭕ | ❌ |
 | **Android** | ❌ | ❌ | ⭕ | ⭕ | ❌ |
 | **Web** | ❌ | ❌ | ❌ | ❌ | 🔷 |
 
+<sup>\* Recent version of Windows 10/11 required for pyke binaries.</sup><br />
+<sup>† glibc ≥ 2.31 (Ubuntu ≥ 20.04) required for pyke binaries.</sup><br />
+<sup>‡ glibc ≥ 2.35 (Ubuntu ≥ 22.04) required for pyke binaries.</sup><br />
+<sup>§ macOS ≥ 10.15 required for pyke binaries.</sup>
+
 If your platform is marked as 🟢 or 🔷, you're in luck! Almost no setup will be required to get `ort` up and running. For platforms marked as ⭕, you'll need to [compile ONNX Runtime from source](https://onnxruntime.ai/docs/build/).
 
 <Note>Certain execution providers may not have binaries available. You can check EP binary support in the [execution providers](/perf/execution-providers) documentation.</Note>
-Original file line number
+Diff line change
@@ Expand Up / @@ -191,3 +191,6 @@ WixTools/ @@
     # IDEA
     .idea
+    # Glassbench results
+    /glassbench*.db