From 070d35497589192e8ae8cdb6f7f49304c3bfb2f5 Mon Sep 17 00:00:00 2001 From: Wanming Lin Date: Wed, 18 Sep 2024 13:04:49 +0800 Subject: [PATCH 1/3] Use MLTensor for Code and Image Classification samples Fixed partial of #275 --- code/.eslintrc.js | 1 + code/samples/matmul.js | 33 +++++++++++-------- code/samples/mul_add.js | 25 +++++++++----- code/samples/simple_graph.js | 23 +++++++++---- image_classification/.eslintrc.js | 1 + .../efficientnet_fp16_nchw.js | 27 +++++++++++---- image_classification/main.js | 12 ++----- image_classification/mobilenet_nchw.js | 27 +++++++++++---- image_classification/mobilenet_nhwc.js | 27 +++++++++++---- image_classification/resnet50v1_fp16_nchw.js | 31 ++++++++++------- image_classification/resnet50v2_nchw.js | 31 ++++++++++------- image_classification/resnet50v2_nhwc.js | 31 ++++++++++------- image_classification/squeezenet_nchw.js | 27 +++++++++++---- image_classification/squeezenet_nhwc.js | 27 +++++++++++---- 14 files changed, 218 insertions(+), 105 deletions(-) diff --git a/code/.eslintrc.js b/code/.eslintrc.js index 3e20ef05..1ddccda7 100644 --- a/code/.eslintrc.js +++ b/code/.eslintrc.js @@ -3,6 +3,7 @@ module.exports = { 'CodeMirror': 'readonly', 'executeCodeSnippet': 'readonly', 'sizeOfShape': 'readonly', + 'MLTensorUsage': 'readonly', }, ignorePatterns: ['libs/'], }; diff --git a/code/samples/matmul.js b/code/samples/matmul.js index 1b773705..41b824aa 100644 --- a/code/samples/matmul.js +++ b/code/samples/matmul.js @@ -1,25 +1,30 @@ // Step 0: Create a context and graph builder for 'gpu', 'cpu' or 'npu'. const context = await navigator.ml.createContext({deviceType: 'gpu'}); const builder = new MLGraphBuilder(context); +const descA = {dataType: 'float32', dimensions: [3, 4], shape: [3, 4]}; +const descB = {dataType: 'float32', dimensions: [4, 3], shape: [4, 3]}; // Step 1: Create a computational graph calculating `c = a * b`. -const a = builder.input('a', { - dataType: 'float32', - dimensions: [3, 4], - shape: [3, 4], -}); -const b = builder.input('b', { - dataType: 'float32', - dimensions: [4, 3], - shape: [4, 3], -}); +const a = builder.input('a', descA); +const b = builder.input('b', descB); const c = builder.matmul(a, b); // Step 2: Compile it into an executable graph. const graph = await builder.build({c}); // Step 3: Bind input and output buffers to the graph and execute. const bufferA = new Float32Array(3*4).fill(1.0); const bufferB = new Float32Array(4*3).fill(0.8); -const bufferC = new Float32Array(3*3); -const results = await context.compute( - graph, {'a': bufferA, 'b': bufferB}, {'c': bufferC}); +descA.usage = MLTensorUsage.WRITE; +descB.usage = MLTensorUsage.WRITE; +const tensorA = await context.createTensor(descA); +const tensorB = await context.createTensor(descB); +context.writeTensor(tensorA, bufferA); +context.writeTensor(tensorB, bufferB); +const tensorC = await context.createTensor({ + dataType: 'float32', + dimensions: [3, 3], + shape: [3, 3], + usage: MLTensorUsage.READ, +}); +context.dispatch(graph, {a: tensorA, b: tensorB}, {c: tensorC}); +const results = await context.readTensor(tensorC); // Step 4: Retrieve the results. -console.log(`values: ${results.outputs.c}`); +console.log(`values: ${new Float32Array(results)}`); diff --git a/code/samples/mul_add.js b/code/samples/mul_add.js index ddfdf512..3ccf0741 100644 --- a/code/samples/mul_add.js +++ b/code/samples/mul_add.js @@ -1,21 +1,30 @@ -const operandType = {dataType: 'float32', dimensions: [2, 2], shape: [2, 2]}; +const desc = {dataType: 'float32', dimensions: [2, 2], shape: [2, 2]}; const context = await navigator.ml.createContext(); const builder = new MLGraphBuilder(context); // 1. Create a computational graph 'C = 0.2 * A + B'. const constant = builder.constant( {dataType: 'float32'}, new Float32Array([0.2])); -const A = builder.input('A', operandType); -const B = builder.input('B', operandType); +const A = builder.input('A', desc); +const B = builder.input('B', desc); const C = builder.add(builder.mul(A, constant), B); // 2. Build the graph into an executable. const graph = await builder.build({'C': C}); // 3. Bind inputs to the graph and execute for the result. const bufferA = new Float32Array(4).fill(1.0); const bufferB = new Float32Array(4).fill(0.8); -const bufferC = new Float32Array(4); -const inputs = {'A': bufferA, 'B': bufferB}; -const outputs = {'C': bufferC}; -const results = await context.compute(graph, inputs, outputs); +desc.usage = MLTensorUsage.WRITE; +const tensorA = await context.createTensor(desc); +const tensorB = await context.createTensor(desc); +context.writeTensor(tensorA, bufferA); +context.writeTensor(tensorB, bufferB); +const tensorC = await context.createTensor({ + ...desc, + usage: MLTensorUsage.READ, +}); +const inputs = {'A': tensorA, 'B': tensorB}; +const outputs = {'C': tensorC}; +context.dispatch(graph, inputs, outputs); // The computed result of [[1, 1], [1, 1]] is in the buffer associated with // the output operand. -console.log('Output value: ' + results.outputs.C); +const results = await context.readTensor(tensorC); +console.log('Output value: ' + new Float32Array(results)); diff --git a/code/samples/simple_graph.js b/code/samples/simple_graph.js index bd0fcb99..8fa2a39d 100644 --- a/code/samples/simple_graph.js +++ b/code/samples/simple_graph.js @@ -50,15 +50,26 @@ const graph = await builder.build({'output': output}); // Setup the input buffers with value 1. const inputBuffer1 = new Float32Array(TENSOR_SIZE).fill(1); const inputBuffer2 = new Float32Array(TENSOR_SIZE).fill(1); -const outputBuffer = new Float32Array(TENSOR_SIZE); + +desc.usage = MLTensorUsage.WRITE; +const inputTensor1 = await context.createTensor(desc); +const inputTensor2 = await context.createTensor(desc); +context.writeTensor(inputTensor1, inputBuffer1); +context.writeTensor(inputTensor2, inputBuffer2); + +const outputTensor = await context.createTensor({ + ...desc, + usage: MLTensorUsage.READ, +}); // Execute the compiled graph with the specified inputs. const inputs = { - 'input1': inputBuffer1, - 'input2': inputBuffer2, + 'input1': inputTensor1, + 'input2': inputTensor2, }; -const outputs = {'output': outputBuffer}; -const results = await context.compute(graph, inputs, outputs); +const outputs = {'output': outputTensor}; +context.dispatch(graph, inputs, outputs); -console.log('Output value: ' + results.outputs.output); +const results = await context.readTensor(outputTensor); +console.log('Output value: ' + new Float32Array(results)); // Output value: 2.25,2.25,2.25,2.25,2.25,2.25,2.25,2.25 diff --git a/image_classification/.eslintrc.js b/image_classification/.eslintrc.js index 41955769..74432882 100644 --- a/image_classification/.eslintrc.js +++ b/image_classification/.eslintrc.js @@ -1,5 +1,6 @@ module.exports = { globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', }, }; diff --git a/image_classification/efficientnet_fp16_nchw.js b/image_classification/efficientnet_fp16_nchw.js index cec1ccd1..5450f0f6 100644 --- a/image_classification/efficientnet_fp16_nchw.js +++ b/image_classification/efficientnet_fp16_nchw.js @@ -8,6 +8,8 @@ export class EfficientNetFP16Nchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/efficientnet_fp16_nchw_optimized/weights/'; this.inputOptions = { @@ -18,7 +20,7 @@ export class EfficientNetFP16Nchw { labelUrl: './labels/labels1000.txt', inputShape: [1, 3, 224, 224], }; - this.outputShape = [1, 1000]; + this.outputShape_ = [1, 1000]; } async buildConv_(input, name, blockName, clip = false, options = {}) { @@ -75,10 +77,19 @@ export class EfficientNetFP16Nchw { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - let data = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + let data = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); data = this.builder_.cast(data, 'float16'); // Block 0 @@ -165,10 +176,12 @@ export class EfficientNetFP16Nchw { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/main.js b/image_classification/main.js index ea3bfedd..8fad561e 100644 --- a/image_classification/main.js +++ b/image_classification/main.js @@ -31,7 +31,6 @@ let loadTime = 0; let buildTime = 0; let computeTime = 0; let inputOptions; -let outputBuffer; let deviceType = ''; let lastdeviceType = ''; let backend = ''; @@ -215,9 +214,8 @@ async function renderCamStream() { const inputCanvas = utils.getVideoFrame(camElement); console.log('- Computing... '); const start = performance.now(); - const results = await netInstance.compute(inputBuffer, outputBuffer); + const outputBuffer = await netInstance.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); - outputBuffer = results.outputs.output; console.log(` done in ${computeTime} ms.`); drawInput(inputCanvas, 'camInCanvas'); showPerfResult(); @@ -334,8 +332,6 @@ async function main() { netInstance = constructNetObject(instanceType); inputOptions = netInstance.inputOptions; labels = await fetchLabels(inputOptions.labelUrl); - outputBuffer = - new Float32Array(utils.sizeOfShape(netInstance.outputShape)); isFirstTimeLoad = false; console.log(`- Model name: ${modelName}, Model layout: ${layout} -`); // UI shows model loading progress @@ -369,12 +365,11 @@ async function main() { let medianComputeTime; // Do warm up - let results = await netInstance.compute(inputBuffer, outputBuffer); + let outputBuffer = await netInstance.compute(inputBuffer); for (let i = 0; i < numRuns; i++) { start = performance.now(); - results = await netInstance.compute( - results.inputs.input, results.outputs.output); + outputBuffer = await netInstance.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` compute time ${i+1}: ${computeTime} ms`); computeTimeArray.push(Number(computeTime)); @@ -384,7 +379,6 @@ async function main() { medianComputeTime = medianComputeTime.toFixed(2); console.log(` median compute time: ${medianComputeTime} ms`); } - outputBuffer = results.outputs.output; console.log('outputBuffer: ', outputBuffer); await ui.showProgressComponent('done', 'done', 'done'); ui.readyShowResultComponents(); diff --git a/image_classification/mobilenet_nchw.js b/image_classification/mobilenet_nchw.js index 014f3e7f..45663233 100644 --- a/image_classification/mobilenet_nchw.js +++ b/image_classification/mobilenet_nchw.js @@ -9,6 +9,8 @@ export class MobileNetV2Nchw { this.deviceType_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.dataType_ = dataType; this.weightsUrl_ = weightsOrigin(); if (this.dataType_ === 'float32') { @@ -27,7 +29,7 @@ export class MobileNetV2Nchw { labelUrl: './labels/labels1000.txt', inputShape: [1, 3, 224, 224], }; - this.outputShape = [1, 1000]; + this.outputShape_ = [1, 1000]; } async buildConv_(input, name, relu6 = true, options = {}) { @@ -89,10 +91,19 @@ export class MobileNetV2Nchw { this.context_ = await navigator.ml.createContext(contextOptions); this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); - let data = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + let data = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); if (this.dataType_ === 'float16') { data = this.builder_.cast(data, 'float16'); @@ -163,10 +174,12 @@ export class MobileNetV2Nchw { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/mobilenet_nhwc.js b/image_classification/mobilenet_nhwc.js index 441eab85..8e3cd632 100644 --- a/image_classification/mobilenet_nhwc.js +++ b/image_classification/mobilenet_nhwc.js @@ -11,6 +11,8 @@ export class MobileNetV2Nhwc { this.deviceType_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/mobilenetv2_nhwc/weights/'; this.inputOptions = { @@ -20,7 +22,7 @@ export class MobileNetV2Nhwc { labelUrl: './labels/labels1001.txt', inputShape: [1, 224, 224, 3], }; - this.outputShape = [1, 1001]; + this.outputShape_ = [1, 1001]; } async buildConv_(input, weightsSubName, biasSubName, relu6, options) { @@ -87,10 +89,19 @@ export class MobileNetV2Nhwc { const strides = [2, 2]; const autoPad = 'same-upper'; const filterLayout = 'ohwi'; - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const conv0 = this.buildConv_( input, '90', 'Conv_Conv2D', true, {strides, autoPad, filterLayout}); @@ -152,10 +163,12 @@ export class MobileNetV2Nhwc { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/resnet50v1_fp16_nchw.js b/image_classification/resnet50v1_fp16_nchw.js index d9cdc843..ec1bee2c 100644 --- a/image_classification/resnet50v1_fp16_nchw.js +++ b/image_classification/resnet50v1_fp16_nchw.js @@ -8,6 +8,8 @@ export class ResNet50V1FP16Nchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/resnet50v1_fp16_nchw_optimized/weights/'; this.inputOptions = { @@ -18,7 +20,7 @@ export class ResNet50V1FP16Nchw { labelUrl: './labels/labels1000.txt', inputShape: [1, 3, 224, 224], }; - this.outputShape = [1, 1000]; + this.outputShape_ = [1, 1000]; } async buildConv_(input, name, stageName, relu, options = undefined) { @@ -76,10 +78,19 @@ export class ResNet50V1FP16Nchw { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - let data = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + let data = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); data = this.builder_.cast(data, 'float16'); const conv1 = await this.buildConv_( @@ -134,14 +145,12 @@ export class ResNet50V1FP16Nchw { } // Release the constant tensors of a model - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute( - await this.graph_, - inputs, - outputs, - ); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/resnet50v2_nchw.js b/image_classification/resnet50v2_nchw.js index 56201dee..3dcfa55c 100644 --- a/image_classification/resnet50v2_nchw.js +++ b/image_classification/resnet50v2_nchw.js @@ -8,6 +8,8 @@ export class ResNet50V2Nchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/resnet50v2_nchw/weights/'; this.inputOptions = { @@ -18,7 +20,7 @@ export class ResNet50V2Nchw { labelUrl: './labels/labels1000.txt', inputShape: [1, 3, 224, 224], }; - this.outputShape = [1, 1000]; + this.outputShape_ = [1, 1000]; } async buildConv_(input, name, stageName, options = undefined) { @@ -98,10 +100,19 @@ export class ResNet50V2Nchw { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const data = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const data = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const bn1 = this.buildBatchNorm_(data, '0', '', false); const conv0 = this.buildConv_( @@ -170,14 +181,12 @@ export class ResNet50V2Nchw { } // Release the constant tensors of a model - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute( - await this.graph_, - inputs, - outputs, - ); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/resnet50v2_nhwc.js b/image_classification/resnet50v2_nhwc.js index 3babf6f3..d1a9ab1d 100644 --- a/image_classification/resnet50v2_nhwc.js +++ b/image_classification/resnet50v2_nhwc.js @@ -12,6 +12,8 @@ export class ResNet50V2Nhwc { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/resnet50v2_nhwc/weights/'; this.inputOptions = { @@ -21,7 +23,7 @@ export class ResNet50V2Nhwc { labelUrl: './labels/labels1001.txt', inputShape: [1, 224, 224, 3], }; - this.outputShape = [1, 1001]; + this.outputShape_ = [1, 1001]; } async buildConv_(input, nameIndices, options = {}, relu = true) { @@ -120,10 +122,19 @@ export class ResNet50V2Nhwc { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const conv1 = await this.buildConv_( input, ['', '', '1'], {strides, padding: [3, 3, 3, 3]}, false); @@ -198,14 +209,12 @@ export class ResNet50V2Nhwc { } // Release the constant tensors of a model - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute( - await this.graph_, - inputs, - outputs, - ); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/squeezenet_nchw.js b/image_classification/squeezenet_nchw.js index 6fe75e5b..5c6f9046 100644 --- a/image_classification/squeezenet_nchw.js +++ b/image_classification/squeezenet_nchw.js @@ -8,6 +8,8 @@ export class SqueezeNetNchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/squeezenet1.1_nchw/weights/'; this.inputOptions = { @@ -18,7 +20,7 @@ export class SqueezeNetNchw { labelUrl: './labels/labels1000.txt', inputShape: [1, 3, 224, 224], }; - this.outputShape = [1, 1000]; + this.outputShape_ = [1, 1000]; } async buildConv_(input, name, options = {}) { @@ -43,10 +45,19 @@ export class SqueezeNetNchw { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const data = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const data = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const conv0 = this.buildConv_(data, 'conv0', {strides: [2, 2]}); const pool0 = this.builder_.maxPool2d( @@ -82,10 +93,12 @@ export class SqueezeNetNchw { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/image_classification/squeezenet_nhwc.js b/image_classification/squeezenet_nhwc.js index c9d7d2d1..093bc15b 100644 --- a/image_classification/squeezenet_nhwc.js +++ b/image_classification/squeezenet_nhwc.js @@ -8,6 +8,8 @@ export class SqueezeNetNhwc { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/squeezenet1.0_nhwc/weights/'; this.inputOptions = { @@ -17,7 +19,7 @@ export class SqueezeNetNhwc { labelUrl: './labels/labels1001.txt', inputShape: [1, 224, 224, 3], }; - this.outputShape = [1, 1001]; + this.outputShape_ = [1, 1001]; } async buildConv_(input, name, options = {}) { @@ -54,10 +56,19 @@ export class SqueezeNetNhwc { this.builder_ = new MLGraphBuilder(this.context_); const strides = [2, 2]; const layout = 'nhwc'; - const placeholder = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const placeholder = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const conv1 = this.buildConv_( placeholder, 'conv1', {strides, autoPad: 'same-upper'}); @@ -94,10 +105,12 @@ export class SqueezeNetNhwc { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } From 77cafc0506e7182365bded19090cf3599624ad0d Mon Sep 17 00:00:00 2001 From: Wanming Lin Date: Fri, 20 Sep 2024 14:46:46 +0800 Subject: [PATCH 2/3] Update rest samples --- face_recognition/.eslintrc.js | 1 + face_recognition/facenet_nchw.js | 29 ++++-- face_recognition/facenet_nhwc.js | 25 ++++- face_recognition/main.js | 28 ++---- facial_landmark_detection/.eslintrc.js | 1 + .../face_landmark_nchw.js | 25 ++++- .../face_landmark_nhwc.js | 25 ++++- facial_landmark_detection/main.js | 28 ++---- .../ssd_mobilenetv2_face_nchw.js | 28 +++++- .../ssd_mobilenetv2_face_nhwc.js | 28 +++++- lenet/.eslintrc.cjs | 1 + lenet/lenet.js | 27 ++++-- lenet/main.js | 9 +- nnotepad/js/nnotepad.js | 44 ++++++--- nsnet2/.eslintrc.js | 1 + nsnet2/denoiser.js | 10 +- nsnet2/nsnet2.js | 76 ++++++++++----- object_detection/.eslintrc.js | 1 + object_detection/main.js | 28 ++---- object_detection/ssd_mobilenetv1_nchw.js | 38 +++++++- object_detection/ssd_mobilenetv1_nhwc.js | 38 +++++++- object_detection/tiny_yolov2_nchw.js | 27 ++++-- object_detection/tiny_yolov2_nhwc.js | 26 +++-- rnnoise/.eslintrc.js | 1 + rnnoise/main.js | 23 +---- rnnoise/rnnoise.js | 97 ++++++++++++++++--- semantic_segmentation/.eslintrc.js | 1 + semantic_segmentation/deeplabv3_mnv2_nchw.js | 26 +++-- semantic_segmentation/deeplabv3_mnv2_nhwc.js | 27 ++++-- semantic_segmentation/main.js | 18 ++-- style_transfer/.eslintrc.js | 1 + style_transfer/fast_style_transfer_net.js | 26 +++-- style_transfer/main.js | 20 ++-- 33 files changed, 539 insertions(+), 245 deletions(-) diff --git a/face_recognition/.eslintrc.js b/face_recognition/.eslintrc.js index c02d313a..a05e0180 100644 --- a/face_recognition/.eslintrc.js +++ b/face_recognition/.eslintrc.js @@ -1,6 +1,7 @@ module.exports = { globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', 'tf': 'readonly', }, }; diff --git a/face_recognition/facenet_nchw.js b/face_recognition/facenet_nchw.js index 91410492..ed033321 100644 --- a/face_recognition/facenet_nchw.js +++ b/face_recognition/facenet_nchw.js @@ -12,6 +12,8 @@ export class FaceNetNchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/facenet_nchw/weights'; this.inputOptions = { @@ -25,6 +27,7 @@ export class FaceNetNchw { distanceMetric: 'euclidean', threshold: 1.26, }; + this.outputShape_ = [1, 512]; } async buildConv_( @@ -138,10 +141,19 @@ export class FaceNetNchw { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { + dataType: 'float32', + dimensions: this.inputOptions.inputShape, + shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ dataType: 'float32', - dimensions: this.inputOptions.inputShape, - shape: this.inputOptions.inputShape, + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const poolOptions = {windowDimensions: [3, 3], strides}; @@ -272,9 +284,12 @@ export class FaceNetNchw { } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/face_recognition/facenet_nhwc.js b/face_recognition/facenet_nhwc.js index 29214a48..d0d6753b 100644 --- a/face_recognition/facenet_nhwc.js +++ b/face_recognition/facenet_nhwc.js @@ -12,6 +12,8 @@ export class FaceNetNhwc { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/facenet_nhwc/weights'; this.inputOptions = { @@ -25,6 +27,7 @@ export class FaceNetNhwc { distanceMetric: 'euclidean', threshold: 1.26, }; + this.outputShape_ = [1, 512]; } async buildConv_(input, namePrefix, options = undefined, relu = true) { @@ -139,10 +142,19 @@ export class FaceNetNhwc { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const poolOptions = {windowDimensions: [3, 3], strides, layout: 'nhwc'}; @@ -241,9 +253,12 @@ export class FaceNetNhwc { } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/face_recognition/main.js b/face_recognition/main.js index ae0c9bc8..103e99ad 100644 --- a/face_recognition/main.js +++ b/face_recognition/main.js @@ -32,8 +32,6 @@ let stream = null; let loadTime = 0; let buildTime = 0; let computeTime = 0; -let fdOutputs; -let frOutputs; let deviceType = ''; let lastdeviceType = ''; let backend = ''; @@ -180,15 +178,14 @@ async function getEmbeddings(inputElem) { const fdInputBuffer = utils.getInputTensor(inputElem, fdInputOptions); let totalComputeTime = 0; let start = performance.now(); - const results = await fdInstance.compute(fdInputBuffer, fdOutputs); + const results = await fdInstance.compute(fdInputBuffer); totalComputeTime = performance.now() - start; - fdOutputs = results.outputs; const strokedRects = []; const embeddings = []; const height = inputElem.naturalHeight || inputElem.height; const width = inputElem.naturalWidth || inputElem.width; const fdOutputArrary = []; - for (const output of Object.entries(fdOutputs)) { + for (const output of Object.entries(results)) { fdOutputArrary.push(output[1]); } const fdSsdOutputs = SsdDecoder.processSsdOutputTensor( @@ -222,10 +219,9 @@ async function getEmbeddings(inputElem) { frInputOptions.drawOptions = drawOptions; const frInputBuffer = utils.getInputTensor(inputElem, frInputOptions); start = performance.now(); - const results = await frInstance.compute(frInputBuffer, frOutputs); + const results = await frInstance.compute(frInputBuffer); totalComputeTime += performance.now() - start; - frOutputs = results.outputs; - const [...normEmbedding] = Float32Array.from(frOutputs.output); + const [...normEmbedding] = Float32Array.from(results); embeddings.push(normEmbedding); } return {computeTime: totalComputeTime, strokedRects, embeddings}; @@ -330,12 +326,6 @@ async function main() { frInstance = constructNetObject(frInstanceType); fdInputOptions = fdInstance.inputOptions; frInputOptions = frInstance.inputOptions; - fdOutputs = {}; - for (const outputInfo of Object.entries(fdInstance.outputsInfo)) { - fdOutputs[outputInfo[0]] = - new Float32Array(utils.sizeOfShape(outputInfo[1])); - } - frOutputs = {'output': new Float32Array(utils.sizeOfShape([1, 512]))}; isFirstTimeLoad = false; console.log(`- Model name: ${fdModelName}, Model layout: ${layout} -`); // UI shows model loading progress @@ -374,12 +364,10 @@ async function main() { let medianComputeTime; console.log('- Computing... '); // Do warm up - const fdResults = await fdInstance.compute(new Float32Array( - utils.sizeOfShape(fdInputOptions.inputShape)), fdOutputs); - const frResults = await frInstance.compute(new Float32Array( - utils.sizeOfShape(frInputOptions.inputShape)), frOutputs); - fdOutputs = fdResults.outputs; - frOutputs = frResults.outputs; + await fdInstance.compute(new Float32Array( + utils.sizeOfShape(fdInputOptions.inputShape))); + await frInstance.compute(new Float32Array( + utils.sizeOfShape(frInputOptions.inputShape))); for (let i = 0; i < numRuns; i++) { if (numRuns > 1) { // clear all predicted embeddings for benckmarking diff --git a/facial_landmark_detection/.eslintrc.js b/facial_landmark_detection/.eslintrc.js index c02d313a..a05e0180 100644 --- a/facial_landmark_detection/.eslintrc.js +++ b/facial_landmark_detection/.eslintrc.js @@ -1,6 +1,7 @@ module.exports = { globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', 'tf': 'readonly', }, }; diff --git a/facial_landmark_detection/face_landmark_nchw.js b/facial_landmark_detection/face_landmark_nchw.js index 6fc42ac1..9cce7ea2 100644 --- a/facial_landmark_detection/face_landmark_nchw.js +++ b/facial_landmark_detection/face_landmark_nchw.js @@ -8,12 +8,15 @@ export class FaceLandmarkNchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/face_landmark_nchw/weights'; this.inputOptions = { inputLayout: 'nchw', inputShape: [1, 3, 128, 128], }; + this.outputShape_ = [1, 136]; } async buildMaxPool2d(input, options) { @@ -69,10 +72,19 @@ export class FaceLandmarkNchw { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const poolOptions = @@ -112,9 +124,12 @@ export class FaceLandmarkNchw { } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/facial_landmark_detection/face_landmark_nhwc.js b/facial_landmark_detection/face_landmark_nhwc.js index 4ff29509..b539ecb4 100644 --- a/facial_landmark_detection/face_landmark_nhwc.js +++ b/facial_landmark_detection/face_landmark_nhwc.js @@ -8,12 +8,15 @@ export class FaceLandmarkNhwc { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/face_landmark_nhwc/weights'; this.inputOptions = { inputLayout: 'nhwc', inputShape: [1, 128, 128, 3], }; + this.outputShape_ = [1, 136]; } async buildMaxPool2d(input, options) { @@ -70,10 +73,19 @@ export class FaceLandmarkNhwc { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const poolOptions = @@ -115,9 +127,12 @@ export class FaceLandmarkNhwc { } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/facial_landmark_detection/main.js b/facial_landmark_detection/main.js index ff32c81c..f88aa643 100644 --- a/facial_landmark_detection/main.js +++ b/facial_landmark_detection/main.js @@ -28,8 +28,6 @@ let stream = null; let loadTime = 0; let buildTime = 0; let computeTime = 0; -let fdOutputs; -let fldOutputs; let deviceType = ''; let lastdeviceType = ''; let backend = ''; @@ -149,15 +147,14 @@ async function predict(inputElement) { const fdInputBuffer = utils.getInputTensor(inputElement, fdInputOptions); let totalComputeTime = 0; let start = performance.now(); - const results = await fdInstance.compute(fdInputBuffer, fdOutputs); + const results = await fdInstance.compute(fdInputBuffer); totalComputeTime += performance.now() - start; - fdOutputs = results.outputs; const strokedRects = []; const keyPoints = []; const height = inputElement.naturalHeight || inputElement.height; const width = inputElement.naturalWidth || inputElement.width; const fdOutputArrary = []; - for (const output of Object.entries(fdOutputs)) { + for (const output of Object.entries(results)) { fdOutputArrary.push(output[1]); } const fdSsdOutputs = SsdDecoder.processSsdOutputTensor( @@ -191,10 +188,9 @@ async function predict(inputElement) { fldInputOptions.drawOptions = drawOptions; const fldInputBuffer = utils.getInputTensor(inputElement, fldInputOptions); start = performance.now(); - const results = await fldInstance.compute(fldInputBuffer, fldOutputs); + const results = await fldInstance.compute(fldInputBuffer); totalComputeTime += performance.now() - start; - fldOutputs = results.outputs; - keyPoints.push(fldOutputs.output.slice()); + keyPoints.push(results.slice()); } return [totalComputeTime.toFixed(2), strokedRects, keyPoints]; } @@ -266,12 +262,6 @@ async function main() { fldInstance = constructNetObject(fldInstanceType); fdInputOptions = fdInstance.inputOptions; fldInputOptions = fldInstance.inputOptions; - fdOutputs = {}; - for (const outputInfo of Object.entries(fdInstance.outputsInfo)) { - fdOutputs[outputInfo[0]] = - new Float32Array(utils.sizeOfShape(outputInfo[1])); - } - fldOutputs = {'output': new Float32Array(utils.sizeOfShape([1, 136]))}; isFirstTimeLoad = false; console.log(`- Model name: ${fdModelName}, Model layout: ${layout} -`); // UI shows model loading progress @@ -312,11 +302,9 @@ async function main() { console.log('- Computing... '); // Do warm up const fdResults = await fdInstance.compute(new Float32Array( - utils.sizeOfShape(fdInputOptions.inputShape)), fdOutputs); + utils.sizeOfShape(fdInputOptions.inputShape))); const fldResults = await fldInstance.compute(new Float32Array( - utils.sizeOfShape(fldInputOptions.inputShape)), fldOutputs); - fdOutputs = fdResults.outputs; - fldOutputs = fldResults.outputs; + utils.sizeOfShape(fldInputOptions.inputShape))); for (let i = 0; i < numRuns; i++) { [computeTime, strokedRects, keyPoints] = await predict(imgElement); console.log(` compute time ${i+1}: ${computeTime} ms`); @@ -327,8 +315,8 @@ async function main() { medianComputeTime = medianComputeTime.toFixed(2); console.log(` median compute time: ${medianComputeTime} ms`); } - console.log('Face Detection model outputs: ', fdOutputs); - console.log('Face Landmark model outputs: ', fldOutputs); + console.log('Face Detection model outputs: ', fdResults); + console.log('Face Landmark model outputs: ', fldResults); await ui.showProgressComponent('done', 'done', 'done'); $('#fps').hide(); ui.readyShowResultComponents(); diff --git a/facial_landmark_detection/ssd_mobilenetv2_face_nchw.js b/facial_landmark_detection/ssd_mobilenetv2_face_nchw.js index 6d335e84..121090aa 100644 --- a/facial_landmark_detection/ssd_mobilenetv2_face_nchw.js +++ b/facial_landmark_detection/ssd_mobilenetv2_face_nchw.js @@ -9,6 +9,8 @@ export class SsdMobilenetV2FaceNchw { this.deviceType_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensors_ = {}; this.weightsUrl_ = weightsOrigin() + '/test-data/models/ssd_mobilenetv2_face_nchw/weights/'; this.inputOptions = { @@ -113,11 +115,22 @@ ${nameArray[1]}`; this.context_ = await navigator.ml.createContext(contextOptions); this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, - }); + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + for (const [key, value] of Object.entries(this.outputsInfo)) { + this.outputTensors_[key] = await this.context_.createTensor({ + dataType: 'float32', + dimensions: value, + shape: value, + usage: MLTensorUsage.READ, + }); + } const bottleneck0 = this.buildLinearBottleneck_( input, '0', false, 32, 'convRelu6'); @@ -238,9 +251,14 @@ ${nameArray[1]}`; } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + this.context_.dispatch(this.graph_, inputs, this.outputTensors_); + const results = {}; + for (const [key, value] of Object.entries(this.outputTensors_)) { + results[key] = new Float32Array(await this.context_.readTensor(value)); + } return results; } } diff --git a/facial_landmark_detection/ssd_mobilenetv2_face_nhwc.js b/facial_landmark_detection/ssd_mobilenetv2_face_nhwc.js index e02d01b9..437ec1ac 100644 --- a/facial_landmark_detection/ssd_mobilenetv2_face_nhwc.js +++ b/facial_landmark_detection/ssd_mobilenetv2_face_nhwc.js @@ -9,6 +9,8 @@ export class SsdMobilenetV2FaceNhwc { this.deviceType_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensors_ = {}; this.weightsUrl_ = weightsOrigin() + '/test-data/models/ssd_mobilenetv2_face_nhwc/weights/'; this.inputOptions = { @@ -125,11 +127,22 @@ ${nameArray[1]}`; this.context_ = await navigator.ml.createContext(contextOptions); this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, - }); + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + for (const [key, value] of Object.entries(this.outputsInfo)) { + this.outputTensors_[key] = await this.context_.createTensor({ + dataType: 'float32', + dimensions: value, + shape: value, + usage: MLTensorUsage.READ, + }); + } const bottleneck0 = this.buildLinearBottleneck_( input, '0', false, 32, 'convRelu6'); @@ -250,9 +263,14 @@ ${nameArray[1]}`; } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + this.context_.dispatch(this.graph_, inputs, this.outputTensors_); + const results = {}; + for (const [key, value] of Object.entries(this.outputTensors_)) { + results[key] = new Float32Array(await this.context_.readTensor(value)); + } return results; } } diff --git a/lenet/.eslintrc.cjs b/lenet/.eslintrc.cjs index c204ab87..65e5eb7e 100644 --- a/lenet/.eslintrc.cjs +++ b/lenet/.eslintrc.cjs @@ -2,6 +2,7 @@ module.exports = { ignorePatterns: ['libs/'], globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', 'BigInt64Array': 'readonly', 'mnist': 'readonly', }, diff --git a/lenet/lenet.js b/lenet/lenet.js index c3852509..21ebaaf2 100644 --- a/lenet/lenet.js +++ b/lenet/lenet.js @@ -10,6 +10,9 @@ export class LeNet { this.graph_ = null; this.builder_ = null; this.layout_ = layout; + this.inputTensor_ = null; + this.outputTensor_ = null; + this.outputShape_ = [1, 10]; this.nchwToNhwcPermutation_ = [0, 2, 3, 1]; this.nhwcToNchwPermutation_ = [0, 3, 1, 2]; this.oihwToOhwiPermutation_ = [0, 2, 3, 1]; @@ -25,12 +28,20 @@ export class LeNet { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); const inputShape = /* nchw */ [1, 1, 28, 28]; - let input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: inputShape, shape: inputShape, + }; + let input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); - if (this.layout_ === 'nhwc') { input = this.builder_.transpose( input, {permutation: this.nchwToNhwcPermutation_}); @@ -168,10 +179,12 @@ export class LeNet { this.graph_ = await this.builder_.build({'output': outputOperand}); } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/lenet/main.js b/lenet/main.js index 39deac1a..13f63288 100644 --- a/lenet/main.js +++ b/lenet/main.js @@ -116,15 +116,13 @@ predictButton.addEventListener('click', async function(e) { let inferenceTime; const inferenceTimeArray = []; const input = getInputFromCanvas(); - let outputBuffer = new Float32Array(utils.sizeOfShape([1, 10])); // Do warm up - let results = await lenet.compute(input, outputBuffer); + const results = await lenet.compute(input); for (let i = 0; i < numRuns; i++) { start = performance.now(); - results = await lenet.compute( - results.inputs.input, results.outputs.output); + await lenet.compute(input); inferenceTime = performance.now() - start; console.log(`execution elapsed time: ${inferenceTime.toFixed(2)} ms`); inferenceTimeArray.push(inferenceTime); @@ -142,8 +140,7 @@ predictButton.addEventListener('click', async function(e) { `${medianInferenceTime.toFixed(2)} ms`; } - outputBuffer = results.outputs.output; - const classes = topK(Array.from(outputBuffer)); + const classes = topK(Array.from(results)); classes.forEach((c, i) => { console.log(`\tlabel: ${c.label}, probability: ${c.prob}%`); const labelElement = document.getElementById(`label${i}`); diff --git a/nnotepad/js/nnotepad.js b/nnotepad/js/nnotepad.js index 5086efda..adf9d2cd 100644 --- a/nnotepad/js/nnotepad.js +++ b/nnotepad/js/nnotepad.js @@ -20,10 +20,10 @@ export class BuildError extends Error { } } -export class ComputeError extends Error { +export class DispatchError extends Error { constructor(message) { super(message); - this.name = 'compute()'; + this.name = 'dispatch()'; } } @@ -42,6 +42,17 @@ class WebNNUtil { return Reflect.construct(ctor, [size]); } + static async tensorForOperand(operand, context) { + const desc = { + dataType: operand.dataType(), + dimensions: operand.shape(), + shape: operand.shape(), + usage: MLTensorUsage.READ, + }; + const tensor = await context.createTensor(desc); + return tensor; + } + static dataTypeToBufferType(type) { switch (type) { case 'int8': @@ -370,7 +381,6 @@ export class NNotepad { serializeLine(line, index === lines.length - 1)) .map((line) => line + ';\n') .join(''); - const AsyncFunction = async function() {}.constructor; return [new AsyncFunction(['_', 'Util'], src), src]; @@ -502,7 +512,7 @@ export class NNotepad { const dims = shape.value.map((expr) => expr.value); const ctor = WebNNUtil.dataTypeToBufferType(dataType.value); return `_.constant({dataType: "${dataType.value}", dimensions: ${ - Util.stringify(dims)}}, shape: ${ + Util.stringify(dims)}, shape: ${ Util.stringify(dims)}}, new ${ ctor.name}(await Util.loadBuffer(${Util.stringify(url.value)})))`; } @@ -519,7 +529,7 @@ export class NNotepad { const ctor = WebNNUtil.dataTypeToBufferType(dataType.value); const len = dims.reduce((a, b) => a * b, 1); return `_.constant({dataType: "${dataType.value}", dimensions: ${ - Util.stringify(dims)}}, shape: ${ + Util.stringify(dims)}, shape: ${ Util.stringify(dims)}}, new ${ ctor.name}(${len}))`; } @@ -538,7 +548,7 @@ export class NNotepad { // Call with the output of `makeBuilderFunc()`. Builds an MLContext and // MLGraphBuilder, executes the function to make an MLGraph, then runs - // compute() on it. The output is mapped. + // dispatch() on it. The output is mapped. static async execBuilderFunction(deviceType, builderFunc) { const context = await navigator.ml.createContext({deviceType}); @@ -563,13 +573,14 @@ export class NNotepad { } const namedOutputs = {}; + const outputTensors = {}; const outputBuffers = {}; - outputOperands.forEach((op, index) => { + outputOperands.forEach(async (op, index) => { const name = `output-${index}`; namedOutputs[name] = op; outputBuffers[name] = WebNNUtil.bufferForOperand(op); + outputTensors[name] = await WebNNUtil.tensorForOperand(op, context); }); - let graph; try { graph = await builder.build(namedOutputs); @@ -577,30 +588,33 @@ export class NNotepad { console.warn(ex); throw new BuildError(`${ex.name} : ${ex.message}`); } - const inputBuffers = {}; + const inputTensors = {}; - let result; try { - result = await context.compute(graph, inputBuffers, outputBuffers); + context.dispatch(graph, inputTensors, outputTensors); } catch (ex) { console.warn(ex); - throw new ComputeError(`${ex.name} : ${ex.message}`); + throw new DispatchError(`${ex.name} : ${ex.message}`); } + for (const name in outputBuffers) { + const buffer = await context.readTensor(outputTensors[name]); + const instance = new outputBuffers[name].constructor(buffer); + outputBuffers[name].set(instance); + }; + function maybeProxyForFloat16Array(array) { return ('proxyForFloat16Array' in self) ? self.proxyForFloat16Array(array) : array; } - // window.result = result; - // console.log(result); return outputOperands.map( (op, index) => ({ dataType: op.dataType(), dimensions: op.shape(), shape: op.shape(), - buffer: maybeProxyForFloat16Array(result.outputs[`output-${index}`]), + buffer: maybeProxyForFloat16Array(outputBuffers[`output-${index}`]), })); } diff --git a/nsnet2/.eslintrc.js b/nsnet2/.eslintrc.js index b738e919..a91c39a0 100644 --- a/nsnet2/.eslintrc.js +++ b/nsnet2/.eslintrc.js @@ -2,6 +2,7 @@ module.exports = { ignorePatterns: ['libs/'], globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', 'tf': 'readonly', }, }; diff --git a/nsnet2/denoiser.js b/nsnet2/denoiser.js index 8637f90e..b70468ab 100644 --- a/nsnet2/denoiser.js +++ b/nsnet2/denoiser.js @@ -94,13 +94,6 @@ export class Denoiser { let initialHiddenState155Buffer = new Float32Array(1 * this.batchSize * this.nsnet.hiddenSize); const outputShape = [this.batchSize, this.frames, this.nsnet.frameSize]; - const gru94Shape = - [this.batchSize, 1, this.batchSize, this.nsnet.hiddenSize]; - const gru157Shape = - [this.batchSize, 1, this.batchSize, this.nsnet.hiddenSize]; - const outputBuffer = new Float32Array(sizeOfShape(outputShape)); - const gru94Buffer = new Float32Array(sizeOfShape(gru94Shape)); - const gru157Buffer = new Float32Array(sizeOfShape(gru157Shape)); for (let frame = 0; !lastIteration; frame += this.frames - overlap * 2) { lastIteration = frame + this.frames + 1 > audioFrames; const audioSize = sizePerFrame * (this.frames + 1); @@ -125,8 +118,7 @@ export class Denoiser { const calcFeatTime = (performance.now() - start).toFixed(2); start = performance.now(); const outputs = await this.nsnet.compute( - inputData, initialHiddenState92Buffer, initialHiddenState155Buffer, - outputBuffer, gru94Buffer, gru157Buffer); + inputData, initialHiddenState92Buffer, initialHiddenState155Buffer); const computeTime = (performance.now() - start).toFixed(2); initialHiddenState92Buffer = outputs.gru94; initialHiddenState155Buffer = outputs.gru157; diff --git a/nsnet2/nsnet2.js b/nsnet2/nsnet2.js index e91a8608..6bae92d8 100644 --- a/nsnet2/nsnet2.js +++ b/nsnet2/nsnet2.js @@ -11,6 +11,12 @@ export class NSNet2 { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.initialState92Tensor_ = null; + this.initialState155Tensor_ = null; + this.outputTensor_ = null; + this.gru94Tensor_ = null; + this.gru157Tensor_ = null; this.frameSize = 161; this.hiddenSize = 400; } @@ -36,29 +42,46 @@ export class NSNet2 { const weight217 = await buildConstantByNpy(this.builder_, baseUrl + '217.npy'); const biasFcOut4 = await buildConstantByNpy(this.builder_, baseUrl + 'fc_out_4_bias.npy'); // Build up the network. - const input = this.builder_.input('input', { - dataType: 'float32', - dimensions: [batchSize, frames, this.frameSize], - shape: [batchSize, frames, this.frameSize], - }); + const inputShape = [batchSize, frames, this.frameSize]; + const inputDesc = {dataType: 'float32', dimensions: inputShape, shape: inputShape}; + const input = this.builder_.input('input', inputDesc); + + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + const relu20 = this.builder_.relu(this.builder_.add(this.builder_.matmul(input, weight172), biasFcIn0)); const transpose31 = this.builder_.transpose(relu20, {permutation: [1, 0, 2]}); - const initialState92 = this.builder_.input('initialState92', { - dataType: 'float32', - dimensions: [1, batchSize, this.hiddenSize], - shape: [1, batchSize, this.hiddenSize], - }); + const initialStateShape = [1, batchSize, this.hiddenSize]; + const initialStateDesc = {dataType: 'float32', dimensions: initialStateShape, shape: initialStateShape}; + const initialState92 = this.builder_.input('initialState92', initialStateDesc); const [gru94, gru93] = this.builder_.gru(transpose31, weight192, recurrentWeight193, frames, this.hiddenSize, {bias: bias194, recurrentBias: recurrentBias194, initialHiddenState: initialState92, returnSequence: true}); // Use reshape to implement squeeze(gru93, {axes: [1]}); const squeeze95Shape = gru93.shape(); squeeze95Shape.splice(1, 1); const squeeze95 = this.builder_.reshape(gru93, squeeze95Shape); - const initialState155 = this.builder_.input('initialState155', { + const initialState155 = this.builder_.input('initialState155', initialStateDesc); + + initialStateDesc.usage = MLTensorUsage.WRITE; + this.initialState92Tensor_ = await this.context_.createTensor(initialStateDesc); + this.initialState155Tensor_ = await this.context_.createTensor(initialStateDesc); + + this.outputTensor_ = await this.context_.createTensor({ dataType: 'float32', - dimensions: [1, batchSize, this.hiddenSize], - shape: [1, batchSize, this.hiddenSize], + dimensions: inputShape, + shape: inputShape, // Same as inputShape. + usage: MLTensorUsage.READ, }); + const gruOutputShape = [1, batchSize, this.hiddenSize]; + const gruOutputDesc = { + dataType: 'float32', + dimensions: gruOutputShape, + shape: gruOutputShape, + usage: MLTensorUsage.READ, + }; + this.gru94Tensor_ = await this.context_.createTensor(gruOutputDesc); + this.gru157Tensor_ = await this.context_.createTensor(gruOutputDesc); + const [gru157, gru156] = this.builder_.gru(squeeze95, weight212, recurrentWeight213, frames, this.hiddenSize, {bias: bias214, recurrentBias: recurrentBias214, initialHiddenState: initialState155, returnSequence: true}); // Use reshape to implement squeeze(gru156, {axes: [1]}); @@ -69,6 +92,7 @@ export class NSNet2 { const relu163 = this.builder_.relu(this.builder_.add(this.builder_.matmul(transpose159, weight215), biasFcOut0)); const relu167 = this.builder_.relu(this.builder_.add(this.builder_.matmul(relu163, weight216), biasFcOut2)); const output = this.builder_.sigmoid(this.builder_.add(this.builder_.matmul(relu167, weight217), biasFcOut4)); + return {output, gru94, gru157}; } @@ -76,18 +100,26 @@ export class NSNet2 { this.graph_ = await this.builder_.build(outputOperand); } - async compute(inputBuffer, initialState92Buffer, initialState155Buffer, outputBuffer, gru94Buffer, gru157Buffer) { + async compute(inputBuffer, initialState92Buffer, initialState155Buffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + this.context_.writeTensor(this.initialState92Tensor_, initialState92Buffer); + this.context_.writeTensor(this.initialState155Tensor_, initialState155Buffer); const inputs = { - 'input': inputBuffer, - 'initialState92': initialState92Buffer, - 'initialState155': initialState155Buffer, + 'input': this.inputTensor_, + 'initialState92': this.initialState92Tensor_, + 'initialState155': this.initialState155Tensor_, }; const outputs = { - 'output': outputBuffer.slice(), - 'gru94': gru94Buffer.slice(), - 'gru157': gru157Buffer.slice(), + 'output': this.outputTensor_, + 'gru94': this.gru94Tensor_, + 'gru157': this.gru157Tensor_, + }; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = { + 'output': new Float32Array(await this.context_.readTensor(this.outputTensor_)), + 'gru94': new Float32Array(await this.context_.readTensor(this.gru94Tensor_)), + 'gru157': new Float32Array(await this.context_.readTensor(this.gru157Tensor_)), }; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results.outputs; + return results; } } diff --git a/object_detection/.eslintrc.js b/object_detection/.eslintrc.js index 41955769..74432882 100644 --- a/object_detection/.eslintrc.js +++ b/object_detection/.eslintrc.js @@ -1,5 +1,6 @@ module.exports = { globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', }, }; diff --git a/object_detection/main.js b/object_detection/main.js index f9f9745e..1d6c0cc4 100644 --- a/object_detection/main.js +++ b/object_detection/main.js @@ -27,7 +27,6 @@ let loadTime = 0; let buildTime = 0; let computeTime = 0; let inputOptions; -let outputs; let deviceType = ''; let lastdeviceType = ''; let backend = ''; @@ -181,12 +180,11 @@ async function renderCamStream() { const inputCanvas = utils.getVideoFrame(camElement); console.log('- Computing... '); const start = performance.now(); - const results = await netInstance.compute(inputBuffer, outputs); - outputs = results.outputs; + const results = await netInstance.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` done in ${computeTime} ms.`); showPerfResult(); - await drawOutput(inputCanvas, outputs, labels); + await drawOutput(inputCanvas, results, labels); $('#fps').text(`${(1000/computeTime).toFixed(0)} FPS`); isRendering = false; if (!stopRender) { @@ -271,17 +269,7 @@ async function main() { netInstance = constructNetObject(instanceType); inputOptions = netInstance.inputOptions; labels = await fetchLabels(inputOptions.labelUrl); - if (modelName.includes('tinyyolov2')) { - outputs = { - 'output': new Float32Array( - utils.sizeOfShape(netInstance.outputShape)), - }; - } else { - outputs = { - 'boxes': new Float32Array(utils.sizeOfShape([1, 1917, 1, 4])), - 'scores': new Float32Array(utils.sizeOfShape([1, 1917, 91])), - }; - } + isFirstTimeLoad = false; console.log(`- Model name: ${modelName}, Model layout: ${layout} -`); // UI shows model loading progress @@ -315,12 +303,11 @@ async function main() { let medianComputeTime; // Do warm up - let results = await netInstance.compute(inputBuffer, outputs); + let results = await netInstance.compute(inputBuffer); for (let i = 0; i < numRuns; i++) { start = performance.now(); - results = await netInstance.compute( - results.inputs.input, results.outputs); + await netInstance.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` compute time ${i+1}: ${computeTime} ms`); computeTimeArray.push(Number(computeTime)); @@ -330,12 +317,11 @@ async function main() { medianComputeTime = medianComputeTime.toFixed(2); console.log(` median compute time: ${medianComputeTime} ms`); } - outputs = results.outputs; - console.log('output: ', outputs); + console.log('output: ', results); await ui.showProgressComponent('done', 'done', 'done'); $('#fps').hide(); ui.readyShowResultComponents(); - await drawOutput(imgElement, outputs, labels); + await drawOutput(imgElement, results, labels); showPerfResult(medianComputeTime); } else if (inputType === 'camera') { stream = await utils.getMediaStream(); diff --git a/object_detection/ssd_mobilenetv1_nchw.js b/object_detection/ssd_mobilenetv1_nchw.js index 067dab6f..d8b83c23 100644 --- a/object_detection/ssd_mobilenetv1_nchw.js +++ b/object_detection/ssd_mobilenetv1_nchw.js @@ -11,6 +11,11 @@ export class SsdMobilenetV1Nchw { this.model_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.boxesTensor_ = null; + this.scoresTensor_ = null; + this.boxesShape_ = [1, 1917, 1, 4]; + this.scoresShape_ = [1, 1917, 91]; this.weightsUrl_ = weightsOrigin() + '/test-data/models/ssd_mobilenetv1_nchw/weights'; // Shares the same bias files with 'nhwc' layout @@ -79,11 +84,27 @@ ${nameArray[1]}_BatchNorm_batchnorm`; this.context_ = await navigator.ml.createContext(contextOptions); this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); - let input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + let input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.boxesTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.boxesShape_, + shape: this.boxesShape_, + usage: MLTensorUsage.READ, + }); + this.scoresTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.scoresShape_, + shape: this.scoresShape_, + usage: MLTensorUsage.READ, }); + if (this.targetDataType_ === 'float16') { input = this.builder_.cast(input, 'float16'); } @@ -269,9 +290,18 @@ ${nameArray[1]}_BatchNorm_batchnorm`; } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = { + 'boxes': this.boxesTensor_, + 'scores': this.scoresTensor_, + }; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = { + 'boxes': new Float32Array(await this.context_.readTensor(this.boxesTensor_)), + 'scores': new Float32Array(await this.context_.readTensor(this.scoresTensor_)), + }; return results; } } diff --git a/object_detection/ssd_mobilenetv1_nhwc.js b/object_detection/ssd_mobilenetv1_nhwc.js index 9fe7d316..a0a9ac1a 100644 --- a/object_detection/ssd_mobilenetv1_nhwc.js +++ b/object_detection/ssd_mobilenetv1_nhwc.js @@ -10,6 +10,11 @@ export class SsdMobilenetV1Nhwc { this.model_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.boxesTensor_ = null; + this.scoresTensor_ = null; + this.boxesShape_ = [1, 1917, 1, 4]; + this.scoresShape_ = [1, 1917, 91]; this.weightsUrl_ = weightsOrigin() + '/test-data/models/ssd_mobilenetv1_nhwc/weights'; this.inputOptions = { @@ -85,11 +90,27 @@ ${nameArray[1]}_BatchNorm_batchnorm`; this.context_ = await navigator.ml.createContext(contextOptions); this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.boxesTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.boxesShape_, + shape: this.boxesShape_, + usage: MLTensorUsage.READ, + }); + this.scoresTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.scoresShape_, + shape: this.scoresShape_, + usage: MLTensorUsage.READ, }); + const strides = [2, 2]; const conv0 = await this.buildConv_( input, ['', '0', '', '165__cf__168'], true, {strides}); @@ -246,9 +267,18 @@ ${nameArray[1]}_BatchNorm_batchnorm`; } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = { + 'boxes': this.boxesTensor_, + 'scores': this.scoresTensor_, + }; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = { + 'boxes': new Float32Array(await this.context_.readTensor(this.boxesTensor_)), + 'scores': new Float32Array(await this.context_.readTensor(this.scoresTensor_)), + }; return results; } } diff --git a/object_detection/tiny_yolov2_nchw.js b/object_detection/tiny_yolov2_nchw.js index d6bd91dc..e1b6da18 100644 --- a/object_detection/tiny_yolov2_nchw.js +++ b/object_detection/tiny_yolov2_nchw.js @@ -8,6 +8,8 @@ export class TinyYoloV2Nchw { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.deviceType_ = null; this.targetDataType_ = dataType; this.weightsUrl_ = weightsOrigin() + @@ -19,7 +21,7 @@ export class TinyYoloV2Nchw { anchors: [1.08, 1.19, 3.42, 4.41, 6.63, 11.38, 9.42, 5.11, 16.62, 10.52], inputShape: [1, 3, 416, 416], }; - this.outputShape = [1, 125, 13, 13]; + this.outputShape_ = [1, 13, 13, 125]; } async buildConv_(input, name) { @@ -61,11 +63,21 @@ export class TinyYoloV2Nchw { this.context_ = await navigator.ml.createContext(contextOptions); this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); - let image = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + let image = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); + let mulScale = this.builder_.constant( {dataType: 'float32', dimensions: [1], shape: [1]}, new Float32Array([0.003921568859368563]), @@ -115,9 +127,12 @@ export class TinyYoloV2Nchw { } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return {'output': new Float32Array(results)}; } } diff --git a/object_detection/tiny_yolov2_nhwc.js b/object_detection/tiny_yolov2_nhwc.js index 1be3cd95..af0da7f6 100644 --- a/object_detection/tiny_yolov2_nhwc.js +++ b/object_detection/tiny_yolov2_nhwc.js @@ -8,6 +8,8 @@ export class TinyYoloV2Nhwc { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/tiny_yolov2_nhwc/weights/'; this.inputOptions = { @@ -18,7 +20,7 @@ export class TinyYoloV2Nhwc { inputShape: [1, 416, 416, 3], norm: true, }; - this.outputShape = [1, 13, 13, 125]; + this.outputShape_ = [1, 13, 13, 125]; } async buildConv_(input, name, leakyRelu = true) { @@ -55,10 +57,19 @@ export class TinyYoloV2Nhwc { async load(contextOptions) { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape_, + shape: this.outputShape_, + usage: MLTensorUsage.READ, }); const poolOptions = { @@ -96,9 +107,12 @@ export class TinyYoloV2Nhwc { } } - async compute(inputBuffer, outputs) { - const inputs = {'input': inputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return {'output': new Float32Array(results)}; } } diff --git a/rnnoise/.eslintrc.js b/rnnoise/.eslintrc.js index 96ef1071..ddb387c4 100644 --- a/rnnoise/.eslintrc.js +++ b/rnnoise/.eslintrc.js @@ -2,6 +2,7 @@ module.exports = { ignorePatterns: ['process/', 'utils/'], globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', 'Module': 'readonly', }, }; diff --git a/rnnoise/main.js b/rnnoise/main.js index 624bef9b..a575a01f 100644 --- a/rnnoise/main.js +++ b/rnnoise/main.js @@ -6,7 +6,6 @@ import {addAlert} from '../common/ui.js'; const batchSize = 1; const frames = 100; // Frames is fixed at 100 const frameSize = 480; -const gainsSize = 22; const weightsUrl = utils.weightsOrigin() + '/test-data/models/rnnoise/weights/'; const rnnoise = new RNNoise(weightsUrl, batchSize, frames); @@ -102,20 +101,6 @@ async function denoise() { 'noiseGruInitialH': noiseInitialHiddenStateBuffer, 'denoiseGruInitialH': denoiseInitialHiddenStateBuffer, }; - const outputBuffer = new Float32Array(batchSize * frames * gainsSize); - const vadGruYHBuffer = new Float32Array( - rnnoise.vadGruNumDirections * batchSize * rnnoise.vadGruHiddenSize); - const noiseGruYHBuffer = new Float32Array( - rnnoise.noiseGruNumDirections * batchSize * rnnoise.noiseGruHiddenSize); - const denoiseGruYHBuffer = new Float32Array( - rnnoise.denoiseGruNumDirections * batchSize * - rnnoise.denoiseGruHiddenSize); - let outputs = { - 'denoiseOutput': outputBuffer, - 'vadGruYH': vadGruYHBuffer, - 'noiseGruYH': noiseGruYHBuffer, - 'denoiseGruYH': denoiseGruYHBuffer, - }; if (audioContext.state != 'running') { audioContext.resume().then(function() { @@ -144,11 +129,11 @@ async function denoise() { const preProcessingTime = (performance.now() - start).toFixed(2); inputs.input = new Float32Array(features); start = performance.now(); - outputs = await rnnoise.compute(inputs, outputs); + const outputs = await rnnoise.compute(inputs); const executionTime = (performance.now() - start).toFixed(2); - inputs.vadGruInitialH = outputs.vadGruYH.slice(); - inputs.noiseGruInitialH = outputs.noiseGruYH.slice(); - inputs.denoiseGruInitialH = outputs.denoiseGruYH.slice(); + inputs.vadGruInitialH = outputs.vadGruYH; + inputs.noiseGruInitialH = outputs.noiseGruYH; + inputs.denoiseGruInitialH = outputs.denoiseGruYH; start = performance.now(); const output = analyser.postProcessing(outputs.denoiseOutput); diff --git a/rnnoise/rnnoise.js b/rnnoise/rnnoise.js index bde6e0d0..86ed6cf5 100644 --- a/rnnoise/rnnoise.js +++ b/rnnoise/rnnoise.js @@ -7,10 +7,19 @@ export class RNNoise { this.baseUrl_ = modelPath; this.batchSize_ = batchSize; this.frames_ = frames; + this.gainsSize_ = 22; this.model_ = null; this.context_ = null; this.graph_ = null; this.builder_ = null; + this.inputTensor_ = null; + this.vadGruInitialHTensor_ = null; + this.noiseGruInitialHTensor_ = null; + this.denoiseGruInitialHTensor_ = null; + this.denoiseOutputTensor_ = null; + this.vadGruYHTensor_ = null; + this.noiseGruYHTensor_ = null; + this.denoiseGruYHTensor_ = null; this.featureSize = 42; this.vadGruHiddenSize = 24; this.vadGruNumDirections = 1; @@ -51,11 +60,15 @@ export class RNNoise { const denoiseOutputBias0 = await buildConstantByNpy(this.builder_, this.baseUrl_ + 'denoise_output_bias_0.npy'); // Build up the network. - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: [this.batchSize_, this.frames_, this.featureSize], shape: [this.batchSize_, this.frames_, this.featureSize], - }); + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + const inputDense0 = this.builder_.matmul(input, inputDenseKernel0); const biasedTensorName2 = this.builder_.add(inputDense0, inputDenseBias0); const inputDenseTanh0 = this.builder_.tanh(biasedTensorName2); @@ -67,11 +80,16 @@ export class RNNoise { vadGruBData, [0, 3 * this.vadGruHiddenSize], [1, 3 * this.vadGruHiddenSize]); - const vadGruInitialH = this.builder_.input('vadGruInitialH', { + + const vadGruInitialHDesc = { dataType: 'float32', dimensions: [1, this.batchSize_, this.vadGruHiddenSize], shape: [1, this.batchSize_, this.vadGruHiddenSize], - }); + }; + const vadGruInitialH = this.builder_.input('vadGruInitialH', vadGruInitialHDesc); + vadGruInitialHDesc.usage = MLTensorUsage.WRITE; + this.vadGruInitialHTensor_ = await this.context_.createTensor(vadGruInitialHDesc); + const [vadGruYH, vadGruY] = this.builder_.gru(vadGruX, vadGruW, vadGruR, this.frames_, this.vadGruHiddenSize, { bias: vadGruB, @@ -95,11 +113,16 @@ export class RNNoise { noiseGruBData, [0, 3 * this.noiseGruHiddenSize], [1, 3 * this.noiseGruHiddenSize]); - const noiseGruInitialH = this.builder_.input('noiseGruInitialH', { + + const noiseGruInitialHDesc = { dataType: 'float32', dimensions: [1, this.batchSize_, this.noiseGruHiddenSize], shape: [1, this.batchSize_, this.noiseGruHiddenSize], - }); + }; + const noiseGruInitialH = this.builder_.input('noiseGruInitialH', noiseGruInitialHDesc); + noiseGruInitialHDesc.usage = MLTensorUsage.WRITE; + this.noiseGruInitialHTensor_ = await this.context_.createTensor(noiseGruInitialHDesc); + const [noiseGruYH, noiseGruY] = this.builder_.gru(noiseGruX, noiseGruW, noiseGruR, this.frames_, this.noiseGruHiddenSize, { bias: noiseGruB, @@ -123,11 +146,16 @@ export class RNNoise { denoiseGruBData, [0, 3 * this.denoiseGruHiddenSize], [1, 3 * this.denoiseGruHiddenSize]); - const denoiseGruInitialH = this.builder_.input('denoiseGruInitialH', { + + const denoiseGruInitialHDesc = { dataType: 'float32', dimensions: [1, this.batchSize_, this.denoiseGruHiddenSize], shape: [1, this.batchSize_, this.denoiseGruHiddenSize], - }); + }; + const denoiseGruInitialH = this.builder_.input('denoiseGruInitialH', denoiseGruInitialHDesc); + denoiseGruInitialHDesc.usage = MLTensorUsage.WRITE; + this.denoiseGruInitialHTensor_ = await this.context_.createTensor(denoiseGruInitialHDesc); + const [denoiseGruYH, denoiseGruY] = this.builder_.gru(denoiseGruX, denoiseGruW, denoiseGruR, this.frames_, this.denoiseGruHiddenSize, { bias: denoiseGruB, @@ -147,6 +175,31 @@ export class RNNoise { denoiseOutput0, denoiseOutputBias0); const denoiseOutput = this.builder_.sigmoid(biasedTensorName); + this.denoiseOutputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: [this.batchSize_, this.frames_, this.gainsSize_], + shape: [this.batchSize_, this.frames_, this.gainsSize_], + usage: MLTensorUsage.READ, + }); + this.vadGruYHTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: [this.vadGruNumDirections, this.batchSize_, this.vadGruHiddenSize], + shape: [this.vadGruNumDirections, this.batchSize_, this.vadGruHiddenSize], + usage: MLTensorUsage.READ, + }); + this.noiseGruYHTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: [this.noiseGruNumDirections, this.batchSize_, this.noiseGruHiddenSize], + shape: [this.noiseGruNumDirections, this.batchSize_, this.noiseGruHiddenSize], + usage: MLTensorUsage.READ, + }); + this.denoiseGruYHTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: [this.denoiseGruNumDirections, this.batchSize_, this.denoiseGruHiddenSize], + shape: [this.denoiseGruNumDirections, this.batchSize_, this.denoiseGruHiddenSize], + usage: MLTensorUsage.READ, + }); + return {denoiseOutput, vadGruYH, noiseGruYH, denoiseGruYH}; } @@ -154,8 +207,30 @@ export class RNNoise { this.graph_ = await this.builder_.build(outputOperand); } - async compute(inputs, outputs) { - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results.outputs; + async compute(inputs) { + this.context_.writeTensor(this.inputTensor_, inputs.input); + this.context_.writeTensor(this.vadGruInitialHTensor_, inputs.vadGruInitialH); + this.context_.writeTensor(this.noiseGruInitialHTensor_, inputs.noiseGruInitialH); + this.context_.writeTensor(this.denoiseGruInitialHTensor_, inputs.denoiseGruInitialH); + const inputTensors = { + 'input': this.inputTensor_, + 'vadGruInitialH': this.vadGruInitialHTensor_, + 'noiseGruInitialH': this.noiseGruInitialHTensor_, + 'denoiseGruInitialH': this.denoiseGruInitialHTensor_, + }; + const outputTensors = { + 'denoiseOutput': this.denoiseOutputTensor_, + 'vadGruYH': this.vadGruYHTensor_, + 'noiseGruYH': this.noiseGruYHTensor_, + 'denoiseGruYH': this.denoiseGruYHTensor_, + }; + this.context_.dispatch(this.graph_, inputTensors, outputTensors); + const results = { + 'denoiseOutput': new Float32Array(await this.context_.readTensor(this.denoiseOutputTensor_)), + 'vadGruYH': new Float32Array(await this.context_.readTensor(this.vadGruYHTensor_)), + 'noiseGruYH': new Float32Array(await this.context_.readTensor(this.noiseGruYHTensor_)), + 'denoiseGruYH': new Float32Array(await this.context_.readTensor(this.denoiseGruYHTensor_)), + }; + return results; } } diff --git a/semantic_segmentation/.eslintrc.js b/semantic_segmentation/.eslintrc.js index 788c3263..c99d4448 100644 --- a/semantic_segmentation/.eslintrc.js +++ b/semantic_segmentation/.eslintrc.js @@ -1,6 +1,7 @@ module.exports = { globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', 'iro': 'readonly', 'tf': 'readonly', }, diff --git a/semantic_segmentation/deeplabv3_mnv2_nchw.js b/semantic_segmentation/deeplabv3_mnv2_nchw.js index a0ce7e78..cd310122 100644 --- a/semantic_segmentation/deeplabv3_mnv2_nchw.js +++ b/semantic_segmentation/deeplabv3_mnv2_nchw.js @@ -11,6 +11,8 @@ export class DeepLabV3MNV2Nchw { this.deviceType_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/deeplabv3_mnv2_nchw/weights/'; // Shares the same bias files with 'nhwc' layout @@ -88,11 +90,21 @@ export class DeepLabV3MNV2Nchw { this.builder_ = new MLGraphBuilder(this.context_); const strides = [2, 2]; - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape, + shape: this.outputShape, + usage: MLTensorUsage.READ, }); + const conv0 = this.buildConv_( input, ['MobilenetV2_Conv_Conv2D', '', '551'], 'relu6', {strides, padding: [1, 1, 1, 1]}); const conv1 = this.buildConv_( @@ -161,10 +173,12 @@ export class DeepLabV3MNV2Nchw { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/semantic_segmentation/deeplabv3_mnv2_nhwc.js b/semantic_segmentation/deeplabv3_mnv2_nhwc.js index 69f10eef..aa2218d4 100644 --- a/semantic_segmentation/deeplabv3_mnv2_nhwc.js +++ b/semantic_segmentation/deeplabv3_mnv2_nhwc.js @@ -9,6 +9,8 @@ export class DeepLabV3MNV2Nhwc { this.deviceType_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.weightsUrl_ = weightsOrigin() + '/test-data/models/deeplabv3_mnv2_nhwc/weights/'; this.inputOptions = { @@ -79,11 +81,22 @@ export class DeepLabV3MNV2Nhwc { this.deviceType_ = contextOptions.deviceType; this.builder_ = new MLGraphBuilder(this.context_); const strides = [2, 2]; - const input = this.builder_.input('input', { + + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape, + shape: this.outputShape, + usage: MLTensorUsage.READ, }); + const conv0 = await this.buildConv_( input, 'MobilenetV2_Conv_Conv2D', '', true, {strides}); const conv1 = await this.buildConv_( @@ -152,10 +165,12 @@ export class DeepLabV3MNV2Nhwc { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/semantic_segmentation/main.js b/semantic_segmentation/main.js index df54062f..87b05cbf 100644 --- a/semantic_segmentation/main.js +++ b/semantic_segmentation/main.js @@ -23,7 +23,6 @@ let loadTime = 0; let buildTime = 0; let computeTime = 0; let inputOptions; -let outputBuffer; let renderer; let hoverPos = null; let deviceType = ''; @@ -255,12 +254,11 @@ async function renderCamStream() { const inputCanvas = utils.getVideoFrame(camElement); console.log('- Computing... '); const start = performance.now(); - const results = await netInstance.compute(inputBuffer, outputBuffer); + const outputBuffer = await netInstance.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` done in ${computeTime} ms.`); - outputBuffer = results.outputs.output; showPerfResult(); - await drawOutput(inputCanvas); + await drawOutput(inputCanvas, outputBuffer); $('#fps').text(`${(1000/computeTime).toFixed(0)} FPS`); if ($('.icdisplay').is(':hidden')) { // Ready to show result components @@ -272,7 +270,7 @@ async function renderCamStream() { } } -async function drawOutput(srcElement) { +async function drawOutput(srcElement, outputBuffer) { // TODO: move 'argMax' operation to graph once it is supported in WebNN spec. // https://github.com/webmachinelearning/webnn/issues/184 const [argMaxBuffer, outputShape] = tf.tidy(() => { @@ -353,8 +351,6 @@ export async function main() { netInstance = constructNetObject(instanceType); inputOptions = netInstance.inputOptions; labels = await fetchLabels(inputOptions.labelUrl); - outputBuffer = - new Float32Array(utils.sizeOfShape(netInstance.outputShape)); isFirstTimeLoad = false; console.log(`- Model name: ${modelName}, Model layout: ${layout} -`); // UI shows model loading progress @@ -388,12 +384,11 @@ export async function main() { let medianComputeTime; // Do warm up - let results = await netInstance.compute(inputBuffer, outputBuffer); + const outputBuffer = await netInstance.compute(inputBuffer); for (let i = 0; i < numRuns; i++) { start = performance.now(); - results = await netInstance.compute( - results.inputs.input, results.outputs.output); + await netInstance.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` compute time ${i+1}: ${computeTime} ms`); computeTimeArray.push(Number(computeTime)); @@ -403,12 +398,11 @@ export async function main() { medianComputeTime = medianComputeTime.toFixed(2); console.log(` median compute time: ${medianComputeTime} ms`); } - outputBuffer = results.outputs.output; console.log('output: ', outputBuffer); await ui.showProgressComponent('done', 'done', 'done'); $('#fps').hide(); ui.readyShowResultComponents(); - await drawOutput(imgElement); + await drawOutput(imgElement, outputBuffer); showPerfResult(medianComputeTime); } else if (inputType === 'camera') { stream = await utils.getMediaStream(); diff --git a/style_transfer/.eslintrc.js b/style_transfer/.eslintrc.js index 41955769..74432882 100644 --- a/style_transfer/.eslintrc.js +++ b/style_transfer/.eslintrc.js @@ -1,5 +1,6 @@ module.exports = { globals: { 'MLGraphBuilder': 'readonly', + 'MLTensorUsage': 'readonly', }, }; diff --git a/style_transfer/fast_style_transfer_net.js b/style_transfer/fast_style_transfer_net.js index 14c40f7c..026a1cd4 100644 --- a/style_transfer/fast_style_transfer_net.js +++ b/style_transfer/fast_style_transfer_net.js @@ -10,6 +10,8 @@ export class FastStyleTransferNet { this.context_ = null; this.builder_ = null; this.graph_ = null; + this.inputTensor_ = null; + this.outputTensor_ = null; this.constPow_ = null; this.constAdd_ = null; this.weightsUrl_ = weightsOrigin() + @@ -112,11 +114,21 @@ export class FastStyleTransferNet { new Float32Array([127.5]), ); // Build up the network. - const input = this.builder_.input('input', { + const inputDesc = { dataType: 'float32', dimensions: this.inputOptions.inputShape, shape: this.inputOptions.inputShape, + }; + const input = this.builder_.input('input', inputDesc); + inputDesc.usage = MLTensorUsage.WRITE; + this.inputTensor_ = await this.context_.createTensor(inputDesc); + this.outputTensor_ = await this.context_.createTensor({ + dataType: 'float32', + dimensions: this.outputShape, + shape: this.outputShape, + usage: MLTensorUsage.READ, }); + const conv2D0 = this.builder_.conv2d(this.builder_.pad(input, padding4, padding4, {mode: 'reflection'}), weightConv0); const add0 = this.buildInstanceNormalization_(conv2D0, variableMul0, variableAdd0); @@ -199,10 +211,12 @@ export class FastStyleTransferNet { } } - async compute(inputBuffer, outputBuffer) { - const inputs = {'input': inputBuffer}; - const outputs = {'output': outputBuffer}; - const results = await this.context_.compute(this.graph_, inputs, outputs); - return results; + async compute(inputBuffer) { + this.context_.writeTensor(this.inputTensor_, inputBuffer); + const inputs = {'input': this.inputTensor_}; + const outputs = {'output': this.outputTensor_}; + this.context_.dispatch(this.graph_, inputs, outputs); + const results = await this.context_.readTensor(this.outputTensor_); + return new Float32Array(results); } } diff --git a/style_transfer/main.js b/style_transfer/main.js index a4e7c36c..73c80e2f 100644 --- a/style_transfer/main.js +++ b/style_transfer/main.js @@ -19,7 +19,6 @@ let stream = null; let loadTime = 0; let buildTime = 0; let computeTime = 0; -let outputBuffer; let deviceType = ''; let lastdeviceType = ''; let backend = ''; @@ -113,15 +112,14 @@ async function renderCamStream() { const inputCanvas = utils.getVideoFrame(camElement); console.log('- Computing... '); const start = performance.now(); - const results = await fastStyleTransferNet.compute(inputBuffer, outputBuffer); + const outputBuffer = await fastStyleTransferNet.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` done in ${computeTime} ms.`); - outputBuffer = results.outputs.output; camElement.width = camElement.videoWidth; camElement.height = camElement.videoHeight; drawInput(inputCanvas, 'camInCanvas'); showPerfResult(); - drawOutput('camInCanvas', 'camOutCanvas'); + drawOutput('camInCanvas', 'camOutCanvas', outputBuffer); $('#fps').text(`${(1000/computeTime).toFixed(0)} FPS`); rafReq = requestAnimationFrame(renderCamStream); } @@ -138,7 +136,7 @@ function drawInput(srcElement, canvasId) { ctx.drawImage(srcElement, 0, 0, scaledWidth, scaledHeight); } -function drawOutput(inCanvasId, outCanvasId) { +function drawOutput(inCanvasId, outCanvasId, outputBuffer) { const outputSize = fastStyleTransferNet.outputShape; const height = outputSize[2]; const width = outputSize[3]; @@ -209,8 +207,6 @@ export async function main() { fastStyleTransferNet.dispose(); } fastStyleTransferNet = new FastStyleTransferNet(); - outputBuffer = new Float32Array( - utils.sizeOfShape(fastStyleTransferNet.outputShape)); isFirstTimeLoad = false; isModelChanged = false; console.log(`- Model ID: ${modelId} -`); @@ -247,13 +243,11 @@ export async function main() { let medianComputeTime; // Do warm up - let results = await fastStyleTransferNet.compute( - inputBuffer, outputBuffer); + const outputBuffer = await fastStyleTransferNet.compute(inputBuffer); for (let i = 0; i < numRuns; i++) { start = performance.now(); - results = await fastStyleTransferNet.compute( - results.inputs.input, results.outputs.output); + await fastStyleTransferNet.compute(inputBuffer); computeTime = (performance.now() - start).toFixed(2); console.log(` compute time ${i+1}: ${computeTime} ms`); computeTimeArray.push(Number(computeTime)); @@ -263,11 +257,11 @@ export async function main() { medianComputeTime = medianComputeTime.toFixed(2); console.log(` median compute time: ${medianComputeTime} ms`); } - outputBuffer = results.outputs.output; + await ui.showProgressComponent('done', 'done', 'done'); ui.readyShowResultComponents(); drawInput(imgElement, 'inputCanvas'); - drawOutput('inputCanvas', 'outputCanvas'); + drawOutput('inputCanvas', 'outputCanvas', outputBuffer); showPerfResult(medianComputeTime); } else if (inputType === 'camera') { stream = await utils.getMediaStream(); From 718a9709a4711bc542b7eab1a626586ca02b5426 Mon Sep 17 00:00:00 2001 From: Wanming Lin Date: Fri, 20 Sep 2024 15:07:47 +0800 Subject: [PATCH 3/3] Fix lint issues --- face_recognition/facenet_nchw.js | 6 +- nnotepad/.eslintrc.js | 1 + nnotepad/js/nnotepad.js | 8 +-- nsnet2/denoiser.js | 2 +- nsnet2/nsnet2.js | 2 +- object_detection/main.js | 2 +- object_detection/ssd_mobilenetv1_nchw.js | 8 ++- object_detection/ssd_mobilenetv1_nhwc.js | 6 +- rnnoise/rnnoise.js | 70 +++++++++++++------- semantic_segmentation/deeplabv3_mnv2_nchw.js | 2 +- 10 files changed, 68 insertions(+), 39 deletions(-) diff --git a/face_recognition/facenet_nchw.js b/face_recognition/facenet_nchw.js index ed033321..2b0a9057 100644 --- a/face_recognition/facenet_nchw.js +++ b/face_recognition/facenet_nchw.js @@ -142,9 +142,9 @@ export class FaceNetNchw { this.context_ = await navigator.ml.createContext(contextOptions); this.builder_ = new MLGraphBuilder(this.context_); const inputDesc = { - dataType: 'float32', - dimensions: this.inputOptions.inputShape, - shape: this.inputOptions.inputShape, + dataType: 'float32', + dimensions: this.inputOptions.inputShape, + shape: this.inputOptions.inputShape, }; const input = this.builder_.input('input', inputDesc); inputDesc.usage = MLTensorUsage.WRITE; diff --git a/nnotepad/.eslintrc.js b/nnotepad/.eslintrc.js index 329c64ed..23c8ee1f 100644 --- a/nnotepad/.eslintrc.js +++ b/nnotepad/.eslintrc.js @@ -1,4 +1,5 @@ module.exports = { env: {'es6': true, 'browser': true, 'jquery': false, 'node': true}, parserOptions: {ecmaVersion: 2021, sourceType: 'module'}, + globals: {'MLTensorUsage': 'readonly'}, }; diff --git a/nnotepad/js/nnotepad.js b/nnotepad/js/nnotepad.js index adf9d2cd..ee7ed7da 100644 --- a/nnotepad/js/nnotepad.js +++ b/nnotepad/js/nnotepad.js @@ -597,11 +597,11 @@ export class NNotepad { throw new DispatchError(`${ex.name} : ${ex.message}`); } - for (const name in outputBuffers) { + for (const [name, outputBuffer] of Object.entries(outputBuffers)) { const buffer = await context.readTensor(outputTensors[name]); - const instance = new outputBuffers[name].constructor(buffer); - outputBuffers[name].set(instance); - }; + const instance = new outputBuffer.constructor(buffer); + outputBuffer.set(instance); + } function maybeProxyForFloat16Array(array) { return ('proxyForFloat16Array' in self) ? diff --git a/nsnet2/denoiser.js b/nsnet2/denoiser.js index b70468ab..5fe943eb 100644 --- a/nsnet2/denoiser.js +++ b/nsnet2/denoiser.js @@ -1,6 +1,6 @@ import {NSNet2} from './nsnet2.js'; import * as featurelib from './featurelib.js'; -import {sizeOfShape, getUrlParams, weightsOrigin} from '../common/utils.js'; +import {getUrlParams, weightsOrigin} from '../common/utils.js'; export class Denoiser { constructor(batchSize, frames, sampleRate) { diff --git a/nsnet2/nsnet2.js b/nsnet2/nsnet2.js index 6bae92d8..a7cb1445 100644 --- a/nsnet2/nsnet2.js +++ b/nsnet2/nsnet2.js @@ -61,7 +61,7 @@ export class NSNet2 { squeeze95Shape.splice(1, 1); const squeeze95 = this.builder_.reshape(gru93, squeeze95Shape); const initialState155 = this.builder_.input('initialState155', initialStateDesc); - + initialStateDesc.usage = MLTensorUsage.WRITE; this.initialState92Tensor_ = await this.context_.createTensor(initialStateDesc); this.initialState155Tensor_ = await this.context_.createTensor(initialStateDesc); diff --git a/object_detection/main.js b/object_detection/main.js index 1d6c0cc4..4ab1b424 100644 --- a/object_detection/main.js +++ b/object_detection/main.js @@ -303,7 +303,7 @@ async function main() { let medianComputeTime; // Do warm up - let results = await netInstance.compute(inputBuffer); + const results = await netInstance.compute(inputBuffer); for (let i = 0; i < numRuns; i++) { start = performance.now(); diff --git a/object_detection/ssd_mobilenetv1_nchw.js b/object_detection/ssd_mobilenetv1_nchw.js index d8b83c23..1b1760ee 100644 --- a/object_detection/ssd_mobilenetv1_nchw.js +++ b/object_detection/ssd_mobilenetv1_nchw.js @@ -104,7 +104,7 @@ ${nameArray[1]}_BatchNorm_batchnorm`; shape: this.scoresShape_, usage: MLTensorUsage.READ, }); - + if (this.targetDataType_ === 'float16') { input = this.builder_.cast(input, 'float16'); } @@ -299,8 +299,10 @@ ${nameArray[1]}_BatchNorm_batchnorm`; }; this.context_.dispatch(this.graph_, inputs, outputs); const results = { - 'boxes': new Float32Array(await this.context_.readTensor(this.boxesTensor_)), - 'scores': new Float32Array(await this.context_.readTensor(this.scoresTensor_)), + boxes: new Float32Array( + await this.context_.readTensor(this.boxesTensor_)), + scores: new Float32Array( + await this.context_.readTensor(this.scoresTensor_)), }; return results; } diff --git a/object_detection/ssd_mobilenetv1_nhwc.js b/object_detection/ssd_mobilenetv1_nhwc.js index a0a9ac1a..ea4a81b2 100644 --- a/object_detection/ssd_mobilenetv1_nhwc.js +++ b/object_detection/ssd_mobilenetv1_nhwc.js @@ -276,8 +276,10 @@ ${nameArray[1]}_BatchNorm_batchnorm`; }; this.context_.dispatch(this.graph_, inputs, outputs); const results = { - 'boxes': new Float32Array(await this.context_.readTensor(this.boxesTensor_)), - 'scores': new Float32Array(await this.context_.readTensor(this.scoresTensor_)), + 'boxes': new Float32Array( + await this.context_.readTensor(this.boxesTensor_)), + 'scores': new Float32Array( + await this.context_.readTensor(this.scoresTensor_)), }; return results; } diff --git a/rnnoise/rnnoise.js b/rnnoise/rnnoise.js index 86ed6cf5..6d79271d 100644 --- a/rnnoise/rnnoise.js +++ b/rnnoise/rnnoise.js @@ -80,15 +80,17 @@ export class RNNoise { vadGruBData, [0, 3 * this.vadGruHiddenSize], [1, 3 * this.vadGruHiddenSize]); - + const vadGruInitialHDesc = { dataType: 'float32', dimensions: [1, this.batchSize_, this.vadGruHiddenSize], shape: [1, this.batchSize_, this.vadGruHiddenSize], }; - const vadGruInitialH = this.builder_.input('vadGruInitialH', vadGruInitialHDesc); + const vadGruInitialH = this.builder_.input( + 'vadGruInitialH', vadGruInitialHDesc); vadGruInitialHDesc.usage = MLTensorUsage.WRITE; - this.vadGruInitialHTensor_ = await this.context_.createTensor(vadGruInitialHDesc); + this.vadGruInitialHTensor_ = await this.context_.createTensor( + vadGruInitialHDesc); const [vadGruYH, vadGruY] = this.builder_.gru(vadGruX, vadGruW, vadGruR, this.frames_, this.vadGruHiddenSize, { @@ -119,9 +121,11 @@ export class RNNoise { dimensions: [1, this.batchSize_, this.noiseGruHiddenSize], shape: [1, this.batchSize_, this.noiseGruHiddenSize], }; - const noiseGruInitialH = this.builder_.input('noiseGruInitialH', noiseGruInitialHDesc); + const noiseGruInitialH = this.builder_.input( + 'noiseGruInitialH', noiseGruInitialHDesc); noiseGruInitialHDesc.usage = MLTensorUsage.WRITE; - this.noiseGruInitialHTensor_ = await this.context_.createTensor(noiseGruInitialHDesc); + this.noiseGruInitialHTensor_ = await this.context_.createTensor( + noiseGruInitialHDesc); const [noiseGruYH, noiseGruY] = this.builder_.gru(noiseGruX, noiseGruW, noiseGruR, this.frames_, this.noiseGruHiddenSize, { @@ -146,15 +150,17 @@ export class RNNoise { denoiseGruBData, [0, 3 * this.denoiseGruHiddenSize], [1, 3 * this.denoiseGruHiddenSize]); - + const denoiseGruInitialHDesc = { dataType: 'float32', dimensions: [1, this.batchSize_, this.denoiseGruHiddenSize], shape: [1, this.batchSize_, this.denoiseGruHiddenSize], }; - const denoiseGruInitialH = this.builder_.input('denoiseGruInitialH', denoiseGruInitialHDesc); + const denoiseGruInitialH = this.builder_.input( + 'denoiseGruInitialH', denoiseGruInitialHDesc); denoiseGruInitialHDesc.usage = MLTensorUsage.WRITE; - this.denoiseGruInitialHTensor_ = await this.context_.createTensor(denoiseGruInitialHDesc); + this.denoiseGruInitialHTensor_ = await this.context_.createTensor( + denoiseGruInitialHDesc); const [denoiseGruYH, denoiseGruY] = this.builder_.gru(denoiseGruX, denoiseGruW, denoiseGruR, this.frames_, this.denoiseGruHiddenSize, { @@ -175,28 +181,39 @@ export class RNNoise { denoiseOutput0, denoiseOutputBias0); const denoiseOutput = this.builder_.sigmoid(biasedTensorName); + const denoiseOutputShape = + [this.batchSize_, this.frames_, this.gainsSize_]; this.denoiseOutputTensor_ = await this.context_.createTensor({ dataType: 'float32', - dimensions: [this.batchSize_, this.frames_, this.gainsSize_], - shape: [this.batchSize_, this.frames_, this.gainsSize_], + dimensions: denoiseOutputShape, + shape: denoiseOutputShape, usage: MLTensorUsage.READ, }); + const vadGruYHOutputShape = + [this.vadGruNumDirections, this.batchSize_, this.vadGruHiddenSize]; this.vadGruYHTensor_ = await this.context_.createTensor({ dataType: 'float32', - dimensions: [this.vadGruNumDirections, this.batchSize_, this.vadGruHiddenSize], - shape: [this.vadGruNumDirections, this.batchSize_, this.vadGruHiddenSize], + dimensions: vadGruYHOutputShape, + shape: vadGruYHOutputShape, usage: MLTensorUsage.READ, }); + const noiseGruYHOutputShape = + [this.noiseGruNumDirections, this.batchSize_, this.noiseGruHiddenSize]; this.noiseGruYHTensor_ = await this.context_.createTensor({ dataType: 'float32', - dimensions: [this.noiseGruNumDirections, this.batchSize_, this.noiseGruHiddenSize], - shape: [this.noiseGruNumDirections, this.batchSize_, this.noiseGruHiddenSize], + dimensions: noiseGruYHOutputShape, + shape: noiseGruYHOutputShape, usage: MLTensorUsage.READ, }); + const denoiseGruYHOutputShape = [ + this.denoiseGruNumDirections, + this.batchSize_, + this.denoiseGruHiddenSize, + ]; this.denoiseGruYHTensor_ = await this.context_.createTensor({ dataType: 'float32', - dimensions: [this.denoiseGruNumDirections, this.batchSize_, this.denoiseGruHiddenSize], - shape: [this.denoiseGruNumDirections, this.batchSize_, this.denoiseGruHiddenSize], + dimensions: denoiseGruYHOutputShape, + shape: denoiseGruYHOutputShape, usage: MLTensorUsage.READ, }); @@ -209,9 +226,12 @@ export class RNNoise { async compute(inputs) { this.context_.writeTensor(this.inputTensor_, inputs.input); - this.context_.writeTensor(this.vadGruInitialHTensor_, inputs.vadGruInitialH); - this.context_.writeTensor(this.noiseGruInitialHTensor_, inputs.noiseGruInitialH); - this.context_.writeTensor(this.denoiseGruInitialHTensor_, inputs.denoiseGruInitialH); + this.context_.writeTensor( + this.vadGruInitialHTensor_, inputs.vadGruInitialH); + this.context_.writeTensor( + this.noiseGruInitialHTensor_, inputs.noiseGruInitialH); + this.context_.writeTensor( + this.denoiseGruInitialHTensor_, inputs.denoiseGruInitialH); const inputTensors = { 'input': this.inputTensor_, 'vadGruInitialH': this.vadGruInitialHTensor_, @@ -226,10 +246,14 @@ export class RNNoise { }; this.context_.dispatch(this.graph_, inputTensors, outputTensors); const results = { - 'denoiseOutput': new Float32Array(await this.context_.readTensor(this.denoiseOutputTensor_)), - 'vadGruYH': new Float32Array(await this.context_.readTensor(this.vadGruYHTensor_)), - 'noiseGruYH': new Float32Array(await this.context_.readTensor(this.noiseGruYHTensor_)), - 'denoiseGruYH': new Float32Array(await this.context_.readTensor(this.denoiseGruYHTensor_)), + 'denoiseOutput': new Float32Array( + await this.context_.readTensor(this.denoiseOutputTensor_)), + 'vadGruYH': new Float32Array( + await this.context_.readTensor(this.vadGruYHTensor_)), + 'noiseGruYH': new Float32Array( + await this.context_.readTensor(this.noiseGruYHTensor_)), + 'denoiseGruYH': new Float32Array( + await this.context_.readTensor(this.denoiseGruYHTensor_)), }; return results; } diff --git a/semantic_segmentation/deeplabv3_mnv2_nchw.js b/semantic_segmentation/deeplabv3_mnv2_nchw.js index cd310122..353b347b 100644 --- a/semantic_segmentation/deeplabv3_mnv2_nchw.js +++ b/semantic_segmentation/deeplabv3_mnv2_nchw.js @@ -104,7 +104,7 @@ export class DeepLabV3MNV2Nchw { shape: this.outputShape, usage: MLTensorUsage.READ, }); - + const conv0 = this.buildConv_( input, ['MobilenetV2_Conv_Conv2D', '', '551'], 'relu6', {strides, padding: [1, 1, 1, 1]}); const conv1 = this.buildConv_(