From e13cee63a08cc194ece97d15d8afc06824c9dd33 Mon Sep 17 00:00:00 2001 From: Ryosuke Hayashi Date: Sun, 3 Jan 2021 18:32:42 +0900 Subject: [PATCH 1/2] Add LogSoftmax for CPU --- lib/backends/cpu/op-resolve-rules.ts | 2 ++ lib/backends/cpu/ops/log-softmax.ts | 28 +++++++++++++++++++++++ lib/ops/log-softmax.ts | 33 ++++++++++++++++++++++++++++ test/test-suite-whitelist.jsonc | 8 +++++++ 4 files changed, 71 insertions(+) create mode 100644 lib/backends/cpu/ops/log-softmax.ts create mode 100644 lib/ops/log-softmax.ts diff --git a/lib/backends/cpu/op-resolve-rules.ts b/lib/backends/cpu/op-resolve-rules.ts index c77a8b7e..2c1a63b6 100644 --- a/lib/backends/cpu/op-resolve-rules.ts +++ b/lib/backends/cpu/op-resolve-rules.ts @@ -34,6 +34,7 @@ import * as unaryOps from './ops/unary-op'; import {CpuUnaryOp} from './ops/unary-op'; import {CpuUnsqueeze} from './ops/unsqueeze'; import {CpuUpsample, CpuUpsampleV9} from './ops/upsample'; +import {CpuLogSoftmax} from './ops/log-softmax'; export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['Abs', '', '6+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.abs)], @@ -72,6 +73,7 @@ export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['IsNaN', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.isNan, undefined, 'bool')], ['LeakyRelu', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.leakyRelu, unaryOps.leakyReluInitializer)], ['Log', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.log)], + ['LogSoftmax', '', '1+', () => new CpuLogSoftmax()], ['LRN', '', '1+', () => new CpuLrn()], ['MatMul', '', '1+', () => new CpuMatMul()], ['MaxPool', '', '1-9', () => new CpuMaxPool()], // TODO: support new attributes for MaxPool-8 and MaxPool-10 diff --git a/lib/backends/cpu/ops/log-softmax.ts b/lib/backends/cpu/ops/log-softmax.ts new file mode 100644 index 00000000..db3ceb8c --- /dev/null +++ b/lib/backends/cpu/ops/log-softmax.ts @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +import {LogSoftmax} from '../../../ops/log-softmax'; +import {Tensor} from '../../../tensor'; +import {CpuInferenceHandler} from '../inference-handler'; +import {softmax} from './softmax'; + +export class CpuLogSoftmax extends LogSoftmax { + run(inferenceHandler: CpuInferenceHandler, inputs: Tensor[]): Tensor[] { + const output = logSoftmax(inputs[0], this.axis); + return [output]; + } +} + +export function logSoftmax(x: Tensor, axis: number): Tensor { + const y = softmax(x, axis); + const yData = y.numberData; + + const output = new Tensor(x.dims, x.type); + const data = output.numberData; + + for (let i = 0; i < yData.length; ++i) { + data[i] = Math.log(yData[i]); + } + + return output; +} diff --git a/lib/ops/log-softmax.ts b/lib/ops/log-softmax.ts new file mode 100644 index 00000000..fb007644 --- /dev/null +++ b/lib/ops/log-softmax.ts @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +import {Attribute} from '../attribute'; +import {InferenceHandler} from '../backend'; +import {Operator} from '../operators'; +import {Tensor} from '../tensor'; + +export abstract class LogSoftmax implements Operator { + abstract run(inferenceHandler: InferenceHandler, inputs: Tensor[]): Tensor[]|Promise; + + initialize(attributes: Attribute): void { + this.axis = attributes.getInt('axis', 1); + } + + checkInputs(inputs: Tensor[]): boolean { + if (!inputs || inputs.length !== 1) { + return false; + } + + return this.checkInputTypes(inputs); + } + + protected checkInputTypes(inputs: Tensor[]): boolean { + if (inputs[0].type !== 'float32' && inputs[0].type !== 'float64') { + return false; + } + + return true; + } + + protected axis: number; +} diff --git a/test/test-suite-whitelist.jsonc b/test/test-suite-whitelist.jsonc index 9a1d2d63..7763ad1f 100644 --- a/test/test-suite-whitelist.jsonc +++ b/test/test-suite-whitelist.jsonc @@ -101,6 +101,13 @@ "test_leakyrelu", // "test_lrn_default", <-- failing due to low precison. If absolute CPU error threshold is increased from 1e-4 to 1e-2 (100x increase), it passes the test. // "test_lrn", <-- failing due to low precison. If absolute CPU error threshold is increased from 1e-4 to 1e-3 (10x increase), it passes the test. + "test_logsoftmax_example_1", + "test_logsoftmax_large_number", + "test_logsoftmax_axis_0", + "test_logsoftmax_axis_1", + "test_logsoftmax_axis_2", + "test_logsoftmax_negative_axis", + "test_logsoftmax_default_axis", "test_matmul_2d", "test_matmul_3d", "test_matmul_4d", @@ -245,6 +252,7 @@ "image-scaler.jsonc", //"less.jsonc", "log.jsonc", + // "log-softmax.jsonc", "matmul.jsonc", "mul.jsonc", "neg.jsonc", From 4af5aa54ccb2cde7e87a8127baa7fe0d93fd5867 Mon Sep 17 00:00:00 2001 From: Ryosuke Hayashi Date: Thu, 14 Jan 2021 00:17:27 +0900 Subject: [PATCH 2/2] format --- docs/operators.md | 2 +- lib/backends/cpu/op-resolve-rules.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operators.md b/docs/operators.md index 47b30cee..c8b2464b 100644 --- a/docs/operators.md +++ b/docs/operators.md @@ -75,7 +75,7 @@ _This file is automatically generated from the def files via [this script](/tool | [Less](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Less) | | | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-7), [9+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-9) | | [LessOrEqual](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LessOrEqual) | | | | | [Log](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Log) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-6) | | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-6) | -| [LogSoftmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LogSoftmax) | | | | +| [LogSoftmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LogSoftmax) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LogSoftmax-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LogSoftmax-11) | | | | [Loop](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Loop) | | | | | [LpNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LpNormalization) | | | | | [LpPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LpPool) | | | | diff --git a/lib/backends/cpu/op-resolve-rules.ts b/lib/backends/cpu/op-resolve-rules.ts index 2c1a63b6..8efbb380 100644 --- a/lib/backends/cpu/op-resolve-rules.ts +++ b/lib/backends/cpu/op-resolve-rules.ts @@ -17,6 +17,7 @@ import {CpuGather} from './ops/gather'; import {CpuGemm} from './ops/gemm'; import {CpuImageScaler} from './ops/image-scaler'; import {CpuInstanceNormalization} from './ops/instance-normalization'; +import {CpuLogSoftmax} from './ops/log-softmax'; import {CpuLrn} from './ops/lrn'; import {CpuMatMul} from './ops/matmul'; import {CpuPad} from './ops/pad'; @@ -34,7 +35,6 @@ import * as unaryOps from './ops/unary-op'; import {CpuUnaryOp} from './ops/unary-op'; import {CpuUnsqueeze} from './ops/unsqueeze'; import {CpuUpsample, CpuUpsampleV9} from './ops/upsample'; -import {CpuLogSoftmax} from './ops/log-softmax'; export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['Abs', '', '6+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.abs)],