diff --git a/docs/operators.md b/docs/operators.md index 47b30cee..b8fbfd2e 100644 --- a/docs/operators.md +++ b/docs/operators.md @@ -72,7 +72,7 @@ _This file is automatically generated from the def files via [this script](/tool | [LRN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LRN) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LRN-1) | | | | [LSTM](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM) | | | | | [LeakyRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LeakyRelu) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LeakyRelu-6) | | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LeakyRelu-6) | -| [Less](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Less) | | | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-7), [9+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-9) | +| [Less](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Less) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-7), [9+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-9) | | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-7), [9+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-9) | | [LessOrEqual](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LessOrEqual) | | | | | [Log](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Log) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-6) | | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-6) | | [LogSoftmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LogSoftmax) | | | | @@ -123,7 +123,7 @@ _This file is automatically generated from the def files via [this script](/tool | [ReduceSumSquare](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceSumSquare) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-11) | | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-11) | | [Relu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Relu) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-6) | | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-6) | | [Reshape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Reshape) | [5+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-5) | | [5+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-5) | -| [Resize](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Resize) | | | | +| [Resize](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Resize) | [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-10), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-11) | | [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-10), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-11) | | [ReverseSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReverseSequence) | | | | | [RoiAlign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RoiAlign) | | | | | [Round](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Round) | | | | @@ -167,6 +167,6 @@ _This file is automatically generated from the def files via [this script](/tool | [Transpose](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Transpose) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Transpose-1) | | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Transpose-1) | | [Unique](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Unique) | | | | | [Unsqueeze](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Unsqueeze) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-11) | | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-11) | -| [Upsample](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Upsample) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-7), [9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-9) | | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-7) | +| [Upsample](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Upsample) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-7), [9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-9) | | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-7), [9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-9) | | [Where](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Where) | | | | | [Xor](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Xor) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Xor-7) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Xor-7) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Xor-7) | diff --git a/lib/api/tensor-impl-utils.ts b/lib/api/tensor-impl-utils.ts index 6d40f653..d10d87d0 100644 --- a/lib/api/tensor-impl-utils.ts +++ b/lib/api/tensor-impl-utils.ts @@ -15,7 +15,11 @@ export function fromInternalTensor(internalTensor: InternalTensor): ApiTensor { return new ApiTensor(new Float32Array(internalTensor.floatData), 'float32', internalTensor.dims); case 'string': return new ApiTensor(internalTensor.stringData, 'string', internalTensor.dims); - case 'int8' || 'uint8' || 'int16' || 'uint16' || 'uint32': + case 'int8': + case 'uint8': + case 'int16': + case 'uint16': + case 'uint32': return new ApiTensor(new Int32Array(internalTensor.integerData), 'int32', internalTensor.dims); case 'int32': return new ApiTensor(internalTensor.integerData as Int32Array, 'int32', internalTensor.dims); @@ -57,7 +61,7 @@ export function matchElementType(type: TensorInterface.Type, element: TensorInte } export function validateIndices(indices: ReadonlyArray) { - if (indices.length < 0 || indices.length > 6) { + if (indices.length > 6) { throw new RangeError(`Only rank 0 to 6 is supported for tensor shape.`); } for (const n of indices) { diff --git a/lib/backends/cpu/op-resolve-rules.ts b/lib/backends/cpu/op-resolve-rules.ts index c77a8b7e..c1c18fbb 100644 --- a/lib/backends/cpu/op-resolve-rules.ts +++ b/lib/backends/cpu/op-resolve-rules.ts @@ -33,7 +33,7 @@ import {CpuTranspose} from './ops/transpose'; import * as unaryOps from './ops/unary-op'; import {CpuUnaryOp} from './ops/unary-op'; import {CpuUnsqueeze} from './ops/unsqueeze'; -import {CpuUpsample, CpuUpsampleV9} from './ops/upsample'; +import {CpuUpsample} from './ops/upsample'; export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['Abs', '', '6+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.abs)], @@ -71,6 +71,7 @@ export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['InstanceNormalization', '', '6+', () => new CpuInstanceNormalization()], ['IsNaN', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.isNan, undefined, 'bool')], ['LeakyRelu', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.leakyRelu, unaryOps.leakyReluInitializer)], + ['Less', '', '7+', () => new CpuBinaryOp(NUMBER_TYPES, (a, b) => a < b ? 1 : 0, undefined, 'bool')], ['Log', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.log)], ['LRN', '', '1+', () => new CpuLrn()], ['MatMul', '', '1+', () => new CpuMatMul()], @@ -92,6 +93,8 @@ export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['ReduceSumSquare', '', '1+', () => new cpuReduce.CpuReduceSumSquare()], ['Relu', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.relu)], ['Reshape', '', '5+', () => new CpuReshape()], + ['Resize', '', '10', () => new CpuUpsample(10)], + ['Resize', '', '11+', () => new CpuUpsample(11)], ['Shape', '', '1+', () => new CpuShape()], ['Sigmoid', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.sigmoid)], ['Sign', '', '9+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.sign)], @@ -109,7 +112,7 @@ export const CPU_OP_RESOLVE_RULES: ReadonlyArray = [ ['Tile', '', '6+', () => new CpuTile()], ['Transpose', '', '1+', () => new CpuTranspose()], ['Unsqueeze', '', '1+', () => new CpuUnsqueeze()], - ['Upsample', '', '7-8', () => new CpuUpsample()], - ['Upsample', '', '9', () => new CpuUpsampleV9()], + ['Upsample', '', '7-8', () => new CpuUpsample(7)], + ['Upsample', '', '9', () => new CpuUpsample(9)], ['Xor', '', '7+', () => new CpuBinaryOp(['bool'], (e1, e2) => (e1 ^ e2))], ]; diff --git a/lib/backends/cpu/ops/concat.ts b/lib/backends/cpu/ops/concat.ts index d01a4ed4..043ba535 100644 --- a/lib/backends/cpu/ops/concat.ts +++ b/lib/backends/cpu/ops/concat.ts @@ -28,7 +28,7 @@ export function concat(x: Tensor[], axis: number) { // ensure all of the non-concatenated axes match each other // along the way, calculate the shape of the output tensor let concatAxisSize = inputShape[axis]; - const outputShape = new Array(inputShape.length); + const outputShape = inputShape.slice(0); for (let i = 1; i < x.length; i++) { const dataN = x[i]; diff --git a/lib/backends/cpu/ops/reshape.ts b/lib/backends/cpu/ops/reshape.ts index 657dc2f8..50468c1d 100644 --- a/lib/backends/cpu/ops/reshape.ts +++ b/lib/backends/cpu/ops/reshape.ts @@ -16,7 +16,7 @@ export class CpuReshape extends Reshape { export function reshape(x: Tensor, shape: Tensor): Tensor { const reshapedDims = ShapeUtil.calculateReshapedDims(x.dims, shape.integerData); const output = new Tensor(reshapedDims, x.type); - const Y = output.floatData; - Y.set(x.floatData); + const Y = output.numberData; + Y.set(x.numberData); return output; } diff --git a/lib/backends/cpu/ops/upsample.ts b/lib/backends/cpu/ops/upsample.ts index 0a640e9a..9d24f99e 100644 --- a/lib/backends/cpu/ops/upsample.ts +++ b/lib/backends/cpu/ops/upsample.ts @@ -1,104 +1,188 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -import {Upsample, UpsampleV9} from '../../../ops/upsample'; +import {Upsample} from '../../../ops/upsample'; import {Tensor} from '../../../tensor'; import {CpuInferenceHandler} from '../inference-handler'; export class CpuUpsample extends Upsample { run(inferenceHandler: CpuInferenceHandler, inputs: Tensor[]): Tensor[] { - const xDims = inputs[0].dims; - const yDims = xDims.map((dim, i) => Math.floor(dim * this.scales[i])); + const [roi, scales, yDims] = this.prepare(inputs); const y = new Tensor(yDims, inputs[0].type); - if (this.mode === 'nearest') { - upsampleNearest(inputs[0].data, y.data, xDims, yDims, this.scales); - } else { - upsampleLinear(inputs[0].data, y.data, xDims, yDims, this.scales); - } + this.compute(inputs[0], y, roi, scales); return [y]; } -} -export class CpuUpsampleV9 extends UpsampleV9 { - run(inferenceHandler: CpuInferenceHandler, inputs: Tensor[]): Tensor[] { - const scales = inputs[1].floatData; + compute(x: Tensor, y: Tensor, roi: ReadonlyArray, scales: ReadonlyArray): void { + const xDims = x.dims; + const yDims = y.dims; + if (yDims.length !== xDims.length) { + throw new Error('Rank of input and output tensor should be same.'); + } - if (this.mode === 'linear' && scales.length !== 2 && scales.length !== 4) { - throw new Error(`only support 2-D or 4-D upsampling for linear mode`); + if (y.size === 0) { + return; + } + + if (xDims.length !== scales.length) { + throw new Error('input tensor\'s dimension does not match the scales.'); + } + + if (roi.length !== 2 * xDims.length) { + throw new Error('size of roi array should be 2 * N where N is the rank of input tensor X.'); + } + + const noScale = xDims.every((d, i) => yDims[i] === d); + if (noScale) { + y.numberData.set(x.numberData); + return; } - const xDims = inputs[0].dims; - const yDims = xDims.map((dim, i) => Math.floor(dim * scales[i])); - const y = new Tensor(yDims, inputs[0].type); if (this.mode === 'nearest') { - upsampleNearest(inputs[0].data, y.data, xDims, yDims, [...scales]); + upsampleNearest( + x.data, y.data, xDims, yDims, scales, roi, this.isResize, this.useExtrapolation, this.extrapolationValue, + this.useNearest2xOptimization, this.getOriginalCoordinate, this.getNearestPixel); } else { - upsampleLinear(inputs[0].data, y.data, xDims, yDims, [...scales]); + if (xDims.length !== 2 && xDims.length !== 4) { + throw new Error('\'Linear\' mode only support 2-D inputs or 4-D inputs'); + } + const is2D = xDims.length === 2; + const batchSize = is2D ? 1 : xDims[0]; + const numChannels = is2D ? 1 : xDims[1]; + const inputHeight = is2D ? xDims[0] : xDims[2]; + const inputWidth = is2D ? xDims[1] : xDims[3]; + const outputHeight = is2D ? yDims[0] : yDims[2]; + const outputWidth = is2D ? yDims[1] : yDims[3]; + + if (this.mode === 'linear') { + upsampleBilinear( + batchSize, numChannels, inputHeight, inputWidth, outputHeight, outputWidth, is2D ? scales[0] : scales[2], + is2D ? scales[1] : scales[3], roi, this.useExtrapolation, this.extrapolationValue, x.numberData, + y.numberData, this.getOriginalCoordinate); + } else { + upsampleBiCubic( + batchSize, numChannels, inputHeight, inputWidth, outputHeight, outputWidth, is2D ? scales[0] : scales[2], + is2D ? scales[1] : scales[3], this.cubicCoefficientA, this.useExtrapolation, this.extrapolationValue, + this.excludeOutside, roi, x.numberData, y.numberData, this.getOriginalCoordinate); + } } - return [y]; } } function upsampleNearest( xData: Tensor.DataTypeMap[Tensor.DataType], yData: Tensor.DataTypeMap[Tensor.DataType], - xDims: ReadonlyArray, yDims: ReadonlyArray, scales: number[]) { + xDims: ReadonlyArray, yDims: ReadonlyArray, scales: ReadonlyArray, + roi: ReadonlyArray, isResize: boolean, extrapolationEnabled: boolean, extrapolationValue: number, + useNearest2xOptimization: boolean, getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc, + getNearestPixel: Upsample.GetNearestPixelFunc) { const dim = xDims.length; - const inputDimCounter = new Array(dim); - inputDimCounter.fill(0); + if (useNearest2xOptimization && dim === 4 && scales[0] === 1 && scales[1] === 1 && scales[2] === 2 && + scales[3] === 2) { + // TODO: 2x optimization + } + + const inputDimCounter = new Array(dim).fill(0); const inputDimFactor = new Array(dim); + const useExtrapolationValue = new Array(dim); inputDimFactor[dim - 1] = 1; // initialize dimension factor for (let i = dim - 2; i >= 0; i--) { inputDimFactor[i] = inputDimFactor[i + 1] * xDims[i + 1]; } - const outputDimCounter = new Array(dim); - outputDimCounter.fill(0); - outputDimCounter[dim - 1] = -1; let yIdx = 0; let xIdx = 0; - for (; yIdx < yData.length; yIdx++) { - for (let dimIdx = dim - 1; dimIdx >= 0; dimIdx--) { - if (++outputDimCounter[dimIdx] < yDims[dimIdx]) { - let currentInputDimCounter = 0; - const originalIdx = getOriginalCoordinate(outputDimCounter[dimIdx], scales[dimIdx]); - currentInputDimCounter = Math.floor(originalIdx); - currentInputDimCounter = Math.max(0, Math.min(currentInputDimCounter, (xDims[dimIdx] - 1))); - - if (currentInputDimCounter !== inputDimCounter[dimIdx]) { - xIdx += (currentInputDimCounter - inputDimCounter[dimIdx]) * inputDimFactor[dimIdx]; - inputDimCounter[dimIdx] = currentInputDimCounter; + + const oneDimensionProcessor = (dimIdx: number, yDim: number) => { + useExtrapolationValue[dimIdx] = false; + const originalIdx = + getOriginalCoordinate(yDim, scales[dimIdx], yDims[dimIdx], xDims[dimIdx], roi[dimIdx], roi[dim + dimIdx]); + if (extrapolationEnabled && (originalIdx < 0 || originalIdx > xDims[dimIdx] - 1)) { + useExtrapolationValue[dimIdx] = true; + } + let currentInputDimCounter = getNearestPixel(originalIdx, scales[dimIdx] < 1); + currentInputDimCounter = Math.max(0, Math.min(currentInputDimCounter, (xDims[dimIdx] - 1))); + if (currentInputDimCounter !== inputDimCounter[dimIdx]) { + xIdx += (currentInputDimCounter - inputDimCounter[dimIdx]) * inputDimFactor[dimIdx]; + inputDimCounter[dimIdx] = currentInputDimCounter; + } + }; + + if (dim === 1) { + for (let yDim0 = 0; yDim0 < yDims[0]; yDim0++) { + oneDimensionProcessor(0, yDim0); + yData[yIdx++] = useExtrapolationValue[0] ? extrapolationValue : xData[xIdx]; + } + + } else if (dim === 2) { + for (let yDim0 = 0; yDim0 < yDims[0]; yDim0++) { + oneDimensionProcessor(0, yDim0); + for (let yDim1 = 0; yDim1 < yDims[1]; yDim1++) { + oneDimensionProcessor(1, yDim1); + yData[yIdx++] = useExtrapolationValue.some(i => i) ? extrapolationValue : xData[xIdx]; + } + } + + } else if (dim === 3) { + for (let yDim0 = 0; yDim0 < yDims[0]; yDim0++) { + oneDimensionProcessor(0, yDim0); + for (let yDim1 = 0; yDim1 < yDims[1]; yDim1++) { + oneDimensionProcessor(1, yDim1); + for (let yDim2 = 0; yDim2 < yDims[2]; yDim2++) { + oneDimensionProcessor(2, yDim2); + yData[yIdx++] = useExtrapolationValue.some(i => i) ? extrapolationValue : xData[xIdx]; } - break; - } else { - outputDimCounter[dimIdx] = 0; - xIdx += (0 - inputDimCounter[dimIdx]) * inputDimFactor[dimIdx]; - inputDimCounter[dimIdx] = 0; } } - yData[yIdx] = xData[xIdx]; - } -} -function upsampleLinear( - xData: Tensor.DataTypeMap[Tensor.DataType], yData: Tensor.DataTypeMap[Tensor.DataType], - xDims: ReadonlyArray, yDims: ReadonlyArray, scales: number[]) { - const is2D = xDims.length === 2; - const batchSize = is2D ? 1 : xDims[0]; - const numChannels = is2D ? 1 : xDims[1]; - const inputHeight = is2D ? xDims[0] : xDims[2]; - const inputWidth = is2D ? xDims[1] : xDims[3]; - const outputHeight = is2D ? yDims[0] : yDims[2]; - const outputWidth = is2D ? yDims[1] : yDims[3]; - - upsampleBilinear( - xData as Tensor.NumberType, yData as Tensor.NumberType, batchSize, numChannels, inputHeight, inputWidth, - outputHeight, outputWidth, is2D ? scales[0] : scales[2], is2D ? scales[1] : scales[3]); + } else if (dim === 4) { + for (let yDim0 = 0; yDim0 < yDims[0]; yDim0++) { + oneDimensionProcessor(0, yDim0); + for (let yDim1 = 0; yDim1 < yDims[1]; yDim1++) { + oneDimensionProcessor(1, yDim1); + for (let yDim2 = 0; yDim2 < yDims[2]; yDim2++) { + oneDimensionProcessor(2, yDim2); + for (let yDim3 = 0; yDim3 < yDims[3]; yDim3++) { + oneDimensionProcessor(3, yDim3); + yData[yIdx++] = useExtrapolationValue.some(i => i) ? extrapolationValue : xData[xIdx]; + } + } + } + } + } else { + const outputDimCounter = new Array(dim).fill(0); + outputDimCounter[dim - 1] = -1; + + for (; yIdx < yData.length; yIdx++) { + for (let dimIdx = dim - 1; dimIdx >= 0; dimIdx--) { + if (++outputDimCounter[dimIdx] < yDims[dimIdx]) { + let currentInputDimCounter = 0; + const originalIdx = getOriginalCoordinate( + outputDimCounter[dimIdx], scales[dimIdx], yDims[dimIdx], xDims[dimIdx], roi[dimIdx], roi[dim + dimIdx]); + currentInputDimCounter = getNearestPixel(originalIdx, scales[dimIdx] < 1); + currentInputDimCounter = Math.max(0, Math.min(currentInputDimCounter, (xDims[dimIdx] - 1))); + + if (currentInputDimCounter !== inputDimCounter[dimIdx]) { + xIdx += (currentInputDimCounter - inputDimCounter[dimIdx]) * inputDimFactor[dimIdx]; + inputDimCounter[dimIdx] = currentInputDimCounter; + } + break; + } else { + outputDimCounter[dimIdx] = 0; + xIdx += (0 - inputDimCounter[dimIdx]) * inputDimFactor[dimIdx]; + inputDimCounter[dimIdx] = 0; + } + } + yData[yIdx] = xData[xIdx]; + } + } } function upsampleBilinear( - xData: Tensor.NumberType, yData: Tensor.NumberType, batchSize: number, numChannels: number, inputHeight: number, - inputWidth: number, outputHeight: number, outputWidth: number, heightScale: number, widthScale: number) { + batchSize: number, numChannels: number, inputHeight: number, inputWidth: number, outputHeight: number, + outputWidth: number, heightScale: number, widthScale: number, roi: ReadonlyArray, useExtrapolation: boolean, + extrapolationValue: number, xData: Tensor.NumberType, yData: Tensor.NumberType, + getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc) { const yOriginal: number[] = []; const xOriginal: number[] = []; @@ -111,8 +195,10 @@ function upsampleBilinear( const dx1 = new Array(outputWidth); const dx2 = new Array(outputWidth); + const roiYStart = roi.length / 2 - 2; + const roiYEnd = roi.length - 2; for (let y = 0; y < outputHeight; ++y) { - let inY = getOriginalCoordinate(y, heightScale); + let inY = getOriginalCoordinate(y, heightScale, outputHeight, inputHeight, roi[roiYStart], roi[roiYEnd]); yOriginal.push(inY); inY = Math.max(0, Math.min(inY, inputHeight - 1)); @@ -131,8 +217,10 @@ function upsampleBilinear( inputWidthMulY2[y] = inputWidth * inY2; } + const roiXStart = roi.length / 2 - 1; + const roiXEnd = roi.length - 1; for (let x = 0; x < outputWidth; ++x) { - let inX = getOriginalCoordinate(x, widthScale); + let inX = getOriginalCoordinate(x, widthScale, outputWidth, inputWidth, roi[roiXStart], roi[roiXEnd]); xOriginal.push(inX); inX = Math.max(0, Math.min(inX, inputWidth - 1)); @@ -154,6 +242,13 @@ function upsampleBilinear( for (let c = 0; c < numChannels; ++c) { for (let y = 0; y < outputHeight; ++y) { for (let x = 0; x < outputWidth; ++x) { + if (useExtrapolation && + ((yOriginal[y] < 0 || yOriginal[y] > inputHeight - 1) || + (xOriginal[x] < 0 || xOriginal[x] > inputWidth - 1))) { + yData[outputWidth * y + x] = extrapolationValue; + continue; + } + const x11 = xData[xOffset + inputWidthMulY1[y] + inX1[x]]; const x21 = xData[xOffset + inputWidthMulY1[y] + inX2[x]]; const x12 = xData[xOffset + inputWidthMulY2[y] + inX1[x]]; @@ -169,9 +264,166 @@ function upsampleBilinear( } } -function getOriginalCoordinate(xResized: number, xScale: number): number { - // Coordinate transformation mode attr was introduced in version 11, before that asymmetric mode was the only - // available transformation mode - // return ((xResized + 0.5) / xScale) - 0.5; - return xResized / xScale; +const CUBIC_MODE_GRID_LENGTH = 4; + +function getCubicCoeffs(s: number, cubicCoeffA = -0.75): number[] { + s = Math.abs(s); + return [ + (((cubicCoeffA * (s + 1) - 5 * cubicCoeffA) * (s + 1) + 8 * cubicCoeffA) * (s + 1) - 4 * cubicCoeffA), + (((cubicCoeffA + 2) * s - (cubicCoeffA + 3)) * s * s + 1), + (((cubicCoeffA + 2) * (1 - s) - (cubicCoeffA + 3)) * (1 - s) * (1 - s) + 1), + (((cubicCoeffA * (2 - s) - 5 * cubicCoeffA) * (2 - s) + 8 * cubicCoeffA) * (2 - s) - 4 * cubicCoeffA) + ]; +} + +function getDataForCoordinate( + xData: Tensor.NumberType, x: number, y: number, inputHeight: number, inputWidth: number): number { + x = Math.max(0, Math.min(x, inputWidth - 1)); + y = Math.max(0, Math.min(y, inputHeight - 1)); + return xData[y * inputWidth + x]; +} + +function cubicInterpolation1D( + xData: Tensor.NumberType, x: number, y: number, inputHeight: number, inputWidth: number, coeffArray: number[], + coeffSum: number, cache: Map): number { + // When calculating cubic interpolation we move the 4*4 grid across the original data and therefore there is + // opportunity to cache the results for previously seen combinations. + // Check if the result is already available in the cache + const gridStartPosition = (y) * inputWidth + (x - 1); + let result = cache.get(gridStartPosition); + if (result !== undefined) { + return result; + } + + // get the neighbors in 1D and find interpolation for this dimension + // for 1D cubic interpolation 4 samples are used. 2 on the left and 2 on the right of x + result = 0; + for (let i = 0, j = -1; i < CUBIC_MODE_GRID_LENGTH; i++, j++) { + const originalData = getDataForCoordinate(xData, x + j, y, inputHeight, inputWidth); + result += coeffArray[i] / coeffSum * originalData; + } + cache.set(gridStartPosition, result); + + return result; +} + +function upsampleBiCubic( + batchSize: number, numChannels: number, inputHeight: number, inputWidth: number, outputHeight: number, + outputWidth: number, heightScale: number, widthScale: number, cubicCoefficientA: number, useExtrapolation: boolean, + extrapolationValue: number, excludeOutside: boolean, roi: ReadonlyArray, xData: Tensor.NumberType, + yData: Tensor.NumberType, getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc) { + const yOriginal: number[] = []; + const xOriginal: number[] = []; + const cubicCoeffs = new Map(); + const coeffTo1DinterpolationMap = new Map>(); + const roiYStart = roi.length / 2 - 2; + const roiYEnd = roi.length - 2; + const roiXStart = roi.length / 2 - 1; + const roiXEnd = roi.length - 1; + + // generate coefficients in y direction + for (let y = 0; y < outputHeight; ++y) { + const inY = getOriginalCoordinate(y, heightScale, outputHeight, inputHeight, roi[roiYStart], roi[roiYEnd]); + yOriginal.push(inY); + const s = yOriginal[y] - Math.floor(yOriginal[y]); + if (!cubicCoeffs.has(s)) { + cubicCoeffs.set(s, getCubicCoeffs(s, cubicCoefficientA)); + coeffTo1DinterpolationMap.set(s, new Map()); + } + } + + // generate coefficients in x direction + for (let x = 0; x < outputWidth; ++x) { + const inX = getOriginalCoordinate(x, widthScale, outputWidth, inputWidth, roi[roiXStart], roi[roiXEnd]); + xOriginal.push(inX); + const s = xOriginal[x] - Math.floor(xOriginal[x]); + if (!cubicCoeffs.has(s)) { + cubicCoeffs.set(s, getCubicCoeffs(s, cubicCoefficientA)); + coeffTo1DinterpolationMap.set(s, new Map()); + } + } + + // setup up temp arrays to hold coefficients when exclude_outside is set to true + const yCoeffHolder = new Array(CUBIC_MODE_GRID_LENGTH); + const xCoeffHolder = new Array(CUBIC_MODE_GRID_LENGTH); + let yCoeffSum = 1; + let xCoeffSum = 1; + + for (let n = 0; n < batchSize; n++) { + for (let c = 0; c < numChannels; c++) { + for (let y = 0; y < outputHeight; y++) { + const inY = yOriginal[y]; + + // when use_extrapolation is set and original index is out of the dim range + // then use extrapolation_value as the output value. + if (useExtrapolation && (inY < 0 || inY > inputHeight - 1)) { + for (let x = 0; x < outputWidth; x++) { + yData[y * outputWidth + x] = extrapolationValue; + } + continue; + } + + const yInt = Math.floor(inY); + const sY = inY - yInt; + const coeffY = excludeOutside ? yCoeffHolder : cubicCoeffs.get(sY)!; + yCoeffSum = 1; + + if (excludeOutside) { + // When true, the weight of sampling locations outside the grid will be set to 0 + // and the weight will be renormalized so that their sum is 1.0 + yCoeffSum = 0; + const origYCoeffs = cubicCoeffs.get(sY)!; + for (let i = 0, yVal = yInt - 1; yVal <= yInt + 2; yVal++, i++) { + yCoeffHolder[i] = (yVal < 0 || yVal >= inputHeight) ? 0.0 : origYCoeffs[i]; + yCoeffSum += yCoeffHolder[i]; + } + } + + for (let x = 0; x < outputWidth; x++) { + const inX = xOriginal[x]; + + // when use_extrapolation is set and original index is out of the dim range + // then use extrapolation_value as the output value. + if (useExtrapolation && (inX < 0 || inX > inputWidth - 1)) { + yData[y * outputWidth + x] = extrapolationValue; + continue; + } + + const xInt = Math.floor(inX); + const sX = inX - xInt; + const coeffX = excludeOutside ? xCoeffHolder : cubicCoeffs.get(sX)!; + xCoeffSum = 1; + + if (excludeOutside) { + // When true, the weight of sampling locations outside the grid will be set to 0 + // and the weight will be renormalized so that their sum is 1.0 + xCoeffSum = 0; + const origXCoeffs = cubicCoeffs.get(sX)!; + for (let i = 0, xVal = xInt - 1; xVal <= xInt + 2; xVal++, i++) { + xCoeffHolder[i] = (xVal < 0 || xVal >= inputWidth) ? 0.0 : origXCoeffs[i]; + xCoeffSum += xCoeffHolder[i]; + } + } + + // Compute cubic interpolation in x dimension using the x coefficients. + // From the result of cubic interpolation in x dim, compute cubic interpolation in y dimension + const interpolationResultCache = coeffTo1DinterpolationMap.get(sX)!; + let result = 0; + for (let yVal = yInt - 1, i = 0; yVal <= yInt + 2; yVal++, i++) { + const xResult = cubicInterpolation1D( + xData, xInt, yVal, inputHeight, inputWidth, coeffX, xCoeffSum, interpolationResultCache); + result += xResult * coeffY[i] / yCoeffSum; + } + + yData[y * outputWidth + x] = result; + } + } + + xData = xData.subarray(inputHeight * inputWidth); + yData = yData.subarray(outputHeight * outputWidth); + + // clear the cache when moving to the next channel + coeffTo1DinterpolationMap.clear(); + } + } } diff --git a/lib/backends/wasm/op-resolve-rules.ts b/lib/backends/wasm/op-resolve-rules.ts index 92fb6141..4e3eeba4 100644 --- a/lib/backends/wasm/op-resolve-rules.ts +++ b/lib/backends/wasm/op-resolve-rules.ts @@ -15,7 +15,7 @@ import {WasmSoftmax} from './ops/softmax'; import {WasmSum} from './ops/sum'; export const WASM_OP_RESOLVE_RULES: ReadonlyArray = [ - ['Add', '', '7+', () => new WasmBinaryOp(['float32'], 'Add')], + ['Add', '', '7+', () => new WasmBinaryOp(['float32', 'int32'], 'Add')], ['And', '', '7+', () => new WasmBinaryOp(['bool'], 'And')], ['AveragePool', '', '7-10', () => new WasmAveragePool()], // TODO: support new attributes for AveragePool-10 ['BatchNormalization', '', '7+', () => new WasmBatchNormalization()], @@ -29,11 +29,11 @@ export const WASM_OP_RESOLVE_RULES: ReadonlyArray = [ ['InstanceNormalization', '', '6+', () => new WasmInstanceNormalization()], ['MatMul', '', '1+', () => new WasmMatMul()], ['MaxPool', '', '1-9', () => new WasmMaxPool()], // TODO: support new attributes for MaxPool-8 and MaxPool-10 - ['Mul', '', '7+', () => new WasmBinaryOp(['float32'], 'Mul')], + ['Mul', '', '7+', () => new WasmBinaryOp(['float32', 'int32'], 'Mul')], ['Or', '', '7+', () => new WasmBinaryOp(['bool'], 'Or')], ['PRelu', '', '7+', () => new WasmBinaryOp(['float32'], 'PRelu')], ['Softmax', '', '1+', () => new WasmSoftmax()], - ['Sub', '', '7+', () => new WasmBinaryOp(['float32'], 'Sub')], + ['Sub', '', '7+', () => new WasmBinaryOp(['float32', 'int32'], 'Sub')], ['Sum', '', '6+', () => new WasmSum()], // TODO: support multidirectional broadcast for Sum-8 ['Xor', '', '7+', () => new WasmBinaryOp(['bool'], 'Xor')], ]; diff --git a/lib/backends/wasm/ops/binary-op.ts b/lib/backends/wasm/ops/binary-op.ts index 948a0419..59d95b5d 100644 --- a/lib/backends/wasm/ops/binary-op.ts +++ b/lib/backends/wasm/ops/binary-op.ts @@ -25,18 +25,27 @@ export class WasmBinaryOp extends BinaryOp { if (inputs[0].type === 'float32') { fun = '_add_f32'; binaryOpType = 'float32InFloat32Out'; + } else if (inputs[0].type === 'int32') { + fun = '_add_i32'; + binaryOpType = 'int32InInt32Out'; } break; case 'Sub': if (inputs[0].type === 'float32') { fun = '_sub_f32'; binaryOpType = 'float32InFloat32Out'; + } else if (inputs[0].type === 'int32') { + fun = '_sub_i32'; + binaryOpType = 'int32InInt32Out'; } break; case 'Mul': if (inputs[0].type === 'float32') { fun = '_mul_f32'; binaryOpType = 'float32InFloat32Out'; + } else if (inputs[0].type === 'int32') { + fun = '_mul_i32'; + binaryOpType = 'int32InInt32Out'; } break; case 'Div': @@ -74,6 +83,14 @@ export class WasmBinaryOp extends BinaryOp { [inputs[1].floatData, 'float32ptr'], [inputs[1].dims.length, 'int32'], [inputs[1].dims, 'int32ptr'], [result.floatData, 'float32ptr', 'out'], [result.floatData.length, 'int32'], [outputShape.length, 'int32'], [outputShape, 'int32ptr']); + } else if (binaryOpType === 'int32InInt32Out') { + result = new Tensor(outputShape, 'int32'); + WasmBinding.getInstance().ccall( + fun, [inputs[0].integerData as Int32Array, 'int32ptr'], [inputs[0].dims.length, 'int32'], + [inputs[0].dims, 'int32ptr'], [inputs[1].integerData as Int32Array, 'int32ptr'], + [inputs[1].dims.length, 'int32'], [inputs[1].dims, 'int32ptr'], + [result.integerData as Int32Array, 'int32ptr', 'out'], [result.integerData.length, 'int32'], + [outputShape.length, 'int32'], [outputShape, 'int32ptr']); } else if (binaryOpType === 'boolInBoolOut') { result = new Tensor(outputShape, 'bool'); WasmBinding.getInstance().ccall( diff --git a/lib/backends/webgl/op-resolve-rules.ts b/lib/backends/webgl/op-resolve-rules.ts index 7b9d56fe..4ad6edb4 100644 --- a/lib/backends/webgl/op-resolve-rules.ts +++ b/lib/backends/webgl/op-resolve-rules.ts @@ -84,6 +84,8 @@ export const WEBGL_OP_RESOLVE_RULES: ReadonlyArray = [ ['ReduceSumSquare', '', '1+', () => new reduceOps.WebGLReduceSumSquare()], ['Relu', '', '6+', () => new unaryOps.WebGLUnaryOp(FLOAT_TYPES, unaryOps.glslRelu())], ['Reshape', '', '5+', () => new WebGLReshape()], + ['Resize', '', '10', () => new WebGLUpsample(10)], + ['Resize', '', '11+', () => new WebGLUpsample(11)], ['Sigmoid', '', '6+', () => new unaryOps.WebGLUnaryOp(FLOAT_TYPES, unaryOps.glslSigmoid())], ['Sin', '', '7+', () => new unaryOps.WebGLUnaryOp(FLOAT_TYPES, unaryOps.glslSin())], ['Slice', '', '10+', () => new WebGLSliceV10()], // TODO: support 'steps' for Slice-10 @@ -102,7 +104,8 @@ export const WEBGL_OP_RESOLVE_RULES: ReadonlyArray = [ ['Tanh', '', '6+', () => new unaryOps.WebGLUnaryOp(FLOAT_TYPES, unaryOps.glslTanh())], ['Tile', '', '6+', () => new WebGLTile()], ['Transpose', '', '1+', () => new WebGLTranspose()], - ['Upsample', '', '7-8', () => new WebGLUpsample()], + ['Upsample', '', '7-8', () => new WebGLUpsample(7)], + ['Upsample', '', '9', () => new WebGLUpsample(9)], ['Unsqueeze', '', '1+', () => new WebGLUnsqueeze()], ['Xor', '', '7+', () => new binaryOps.WebGLBinaryOp(['bool'], binaryOps.glslXor())], ]; diff --git a/lib/backends/webgl/ops/gather.ts b/lib/backends/webgl/ops/gather.ts index dc67e74f..3ee7aad6 100644 --- a/lib/backends/webgl/ops/gather.ts +++ b/lib/backends/webgl/ops/gather.ts @@ -16,10 +16,6 @@ export class WebGLGather extends Gather implements WebGLOperator { const indexDataShape = inputs[1].dims.slice(); const outputShape = new Array(inputShape.length + indexDataShape.length - 1); - if (outputShape.length === 0) { - throw Error('A scalar tensor output has not been supported'); - } - const axis = ShapeUtil.normalizeAxis(this.axis, inputShape.length); const indexCopyOps: string[] = []; for (let i = 0; i < outputShape.length; i++) { @@ -42,13 +38,14 @@ export class WebGLGather extends Gather implements WebGLOperator { } } - const orank = outputShape.length; + const orank = outputShape.length || 1; const irank = inputShape.length; - const iDrank = indexDataShape.length; + const iDrank = indexDataShape.length || 1; const shaderSource = ` float process(int outputIdx[${orank}]) { int inputIdx[${irank}]; int indexDataIdx[${iDrank}]; + indexDataIdx[0] = 0; ${indexCopyOps.join('\n ')} int idx = int(_B(indexDataIdx)); inputIdx[${axis}] = idx < 0 ? idx + ${inputShape[axis]} : idx; diff --git a/lib/backends/webgl/ops/reshape.ts b/lib/backends/webgl/ops/reshape.ts index 007c3bb6..df42c44b 100644 --- a/lib/backends/webgl/ops/reshape.ts +++ b/lib/backends/webgl/ops/reshape.ts @@ -33,7 +33,6 @@ export function reshape( unpackedShape: reshapedDims, }; - const newTextureData = - inferenceHandler.createSharedTextureData(newTextureLayout, input.type, inputTD.texture, input.dataId); + const newTextureData = inferenceHandler.createSharedTextureData(newTextureLayout, input.type, inputTD.texture, {}); return newTextureData.tensor; } diff --git a/lib/backends/webgl/ops/upsample.ts b/lib/backends/webgl/ops/upsample.ts index 10d6de46..6a7a86a7 100644 --- a/lib/backends/webgl/ops/upsample.ts +++ b/lib/backends/webgl/ops/upsample.ts @@ -3,9 +3,9 @@ import {Upsample} from '../../../ops/upsample'; import {Tensor} from '../../../tensor'; -import {getGlsl} from '../glsl-source'; +import {getGlsl, Glsl} from '../glsl-source'; import {WebGLInferenceHandler} from '../inference-handler'; -import {ProgramInfo, RunData, WebGLOperator} from '../types'; +import {Artifact, ProgramInfo, RunData, TextureLayout, VariableInfo, WebGLOperator} from '../types'; export class WebGLUpsample extends Upsample implements WebGLOperator { run(inferenceHandler: WebGLInferenceHandler, inputs: Tensor[]): Tensor[] { @@ -13,181 +13,528 @@ export class WebGLUpsample extends Upsample implements WebGLOperator { } createProgramInfo(handler: WebGLInferenceHandler, inputs: Tensor[]): ProgramInfo { const inputLayout = handler.getOrCreateTextureLayout(inputs[0]); - const outputShape = inputs[0].dims.map((dim, i) => Math.floor(dim * this.scales[i])); + const [roi, scales, outputShape] = this.prepare(inputs); + this.roiCache = roi; + this.scalesCache = scales.map(x => Math.ceil(x)); const outputLayout = handler.createTextureLayoutFromShape(outputShape); const dim = outputShape.length; const glsl = getGlsl(handler.session.backend.glContext.version); + if (this.isResize) { + this.mappingOriginCache = []; + this.mappingWeightCache = []; + this.mappingExtrapolateCache = []; + this.mappingCoeffCache = []; + return createResizeProgramInfo( + glsl, this.mode, dim, inputLayout, outputLayout, scales, roi, this.useExtrapolation, this.extrapolationValue, + this.cubicCoefficientA, this.excludeOutside, this.coordinateTransformMode, this.getOriginalCoordinate, + this.getNearestPixel, this.mappingOriginCache, this.mappingWeightCache, this.mappingExtrapolateCache, + this.mappingCoeffCache); + } else { + return createUpsampleProgramInfo(glsl, this.mode, dim, inputLayout, outputLayout, scales); + } + } + createRunData(handler: WebGLInferenceHandler, programInfo: ProgramInfo, inputs: Tensor[]): RunData { + const inputTD = handler.getOrCreateTextureData(inputs[0], programInfo.inputLayouts[0]); + const outputTD = handler.createTextureDataFromLayout(programInfo.outputLayout, inputTD.tensor.type); + return { + inputTextureDatas: [inputTD], + outputTextureData: outputTD, + uniformData: { + scales: this.scalesCache, + mo: this.mappingOriginCache, + me: this.mappingExtrapolateCache, + mw: this.mappingWeightCache, + mc: this.mappingCoeffCache + } + }; + } + + protected roiCache: number[]; + protected scalesCache: number[]; + protected mappingOriginCache: number[]; + protected mappingExtrapolateCache: number[]; + protected mappingWeightCache: number[]; + protected mappingCoeffCache: number[]; + + protected artifacts: Artifact[]; +} + +function createResizeProgramInfo( + glsl: Glsl, mode: string, dim: number, inputLayout: TextureLayout, outputLayout: TextureLayout, + scales: ReadonlyArray, roi: ReadonlyArray, extrapolationEnabled: boolean, + extrapolationValue: number, cubicCoefficientA: number, excludeOutside: boolean, coordinateTransformMode: string, + getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc, getNearestPixel: Upsample.GetNearestPixelFunc, + mappingOriginCache: number[], mappingWeightCache: number[], mappingExtrapolateCache: number[], + mappingCoeffCache: number[]): ProgramInfo { + const isSame = scales.every(s => s === 1) && coordinateTransformMode !== 'tf_crop_and_resize'; + if (isSame) { + return { + inputLayouts: [inputLayout], + outputLayout, + samplers: ['X'], + hasMain: true, + shaderSource: `void main() { + vec4 v = ${glsl.texture2D}(X, TexCoords); + ${glsl.output} = v; + }` + }; + } + + const inputShape = inputLayout.shape; + const inputHeight = inputShape[dim - 2]; + const inputWidth = inputShape[dim - 1]; + const outputShape = outputLayout.shape; + const outputHeight = outputShape[dim - 2]; + const outputWidth = outputShape[dim - 1]; + const scalesHeight = scales[dim - 2]; + const scalesWidth = scales[dim - 1]; + const roiStartHeight = roi[dim - 2]; + const roiEndHeight = roi[dim - 2 + dim]; + const roiStartWidth = roi[dim - 1]; + const roiEndWidth = roi[dim - 1 + dim]; + + const precalculatedPitches = shaderPrecalculatedPitches(dim, outputShape, inputShape); + const getInputFloatFunction = shaderGetInputFloatFunction(inputLayout, glsl); - const outputPitches = new Array(dim); - const inputPitches = new Array(dim); - let precalculatedPitches = ` - int output_pitches[${dim}]; - int input_pitches[${dim}]; - `; - for (let d = dim - 1; d >= 0; d--) { - outputPitches[d] = (d === dim - 1) ? 1 : outputPitches[d + 1] * outputShape[d + 1]; - inputPitches[d] = (d === dim - 1) ? 1 : inputPitches[d + 1] * inputs[0].dims[d + 1]; - - precalculatedPitches += ` - output_pitches[${d}] = ${outputPitches[d]}; - input_pitches[${d}] = ${inputPitches[d]}; - `; + if (mode === 'nearest') { + const could2d = + dim >= 2 && coordinateTransformMode !== 'tf_crop_and_resize' && scales.some((s, i) => s === 1 && i < dim - 2); + if (could2d) { + fillResizeNearestMapping2D( + inputHeight, inputWidth, outputHeight, outputWidth, scalesHeight, scalesWidth, roiStartHeight, roiEndHeight, + roiStartWidth, roiEndWidth, extrapolationEnabled, getOriginalCoordinate, getNearestPixel, mappingOriginCache, + mappingExtrapolateCache); + const variables: VariableInfo[] = [{name: 'mo', type: 'int', arrayLength: mappingOriginCache.length}]; + if (extrapolationEnabled) { + variables.push({name: 'me', type: 'int', arrayLength: mappingExtrapolateCache.length}); + } + return { + inputLayouts: [inputLayout], + outputLayout, + samplers: ['X'], + shaderSource: ` + ${getInputFloatFunction} +float process(int indices[${dim}]) { + int input_index = 0; + int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); + + ${precalculatedPitches} + + ${ + dim === 2 ? 'int m = output_index; int imageid = 0;' : + `int imageid = output_index / output_pitches[${ + dim - 3}]; int m = output_index - imageid * output_pitches[${dim - 3}];`} + int h = m / output_pitches[${dim - 2}]; + int w = m - h * output_pitches[${dim - 2}]; + + ${ + extrapolationEnabled ? `if (me[h] + me[${outputHeight}+w] > 0) { + return float(${extrapolationValue}); + }` : + ''} + + input_index = ${dim === 2 ? '0' : `imageid * input_pitches[${dim - 3}]`} + mo[h] * input_pitches[${dim - 2}] + mo[${ + outputHeight}+w]; + return getInputFloat(input_index); +}`, + variables + }; } - const getInputFloatFunction = ` - float getInputFloat(int index) { - vec2 coords = offsetToCoords(index, ${inputLayout.width}, ${inputLayout.height}); - float value = getColorAsFloat(${glsl.texture2D}(X, coords)); - return value; + + // could2d === false + throw new Error('non-2D nearest mode is not implemented yet'); + + } else if (mode === 'linear') { + fillResizeBilinearCoordinateMapping( + inputHeight, inputWidth, outputHeight, outputWidth, scalesHeight, scalesWidth, roiStartHeight, roiEndHeight, + roiStartWidth, roiEndWidth, extrapolationEnabled, getOriginalCoordinate, mappingOriginCache, mappingWeightCache, + mappingExtrapolateCache); + const variables: VariableInfo[] = [ + {name: 'mo', type: 'int', arrayLength: mappingOriginCache.length}, + {name: 'mw', type: 'float', arrayLength: mappingWeightCache.length} + ]; + if (extrapolationEnabled) { + variables.push({name: 'me', type: 'int', arrayLength: mappingExtrapolateCache.length}); } - `; - - const shaderSource = this.mode === 'nearest' ? - // nearest - ` - ${getInputFloatFunction} - float process(int indices[${dim}]) { - int input_index = 0; - int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); - - ${precalculatedPitches} - - int d, m; - for (int dim = 0; dim < ${dim}; ++dim) { - d = output_index / output_pitches[dim]; - m = output_index - d * output_pitches[dim]; - output_index = m; - - if (scales[dim] != 1 && d > 0) { - int d2 = d / scales[dim]; - m = d - d2 * scales[dim]; - d = d2; - } - input_index += input_pitches[dim] * d; - } - - return getInputFloat(input_index); - }` : - dim === 4 ? - // bilinear 4D - ` - ${getInputFloatFunction} - float process(int indices[4]) { - int input_index = 0; - int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); - - ${precalculatedPitches} - - int m; - int index_of_dim0, index_of_dim1, index_of_dim2, index_of_dim3; - index_of_dim0 = output_index / output_pitches[0]; - m = output_index - index_of_dim0 * output_pitches[0]; - index_of_dim1 = m / output_pitches[1]; - m = m - index_of_dim1 * output_pitches[1]; - index_of_dim2 = m / output_pitches[2]; - m = m - index_of_dim2 * output_pitches[2]; - index_of_dim3 = m; - - int index_of_input_dim2, index_of_input_dim3, x_offset, y_offset; - index_of_input_dim2 = index_of_dim2 / scales[2]; - y_offset = index_of_dim2 - index_of_input_dim2 * scales[2]; - index_of_input_dim3 = index_of_dim3 / scales[3]; - x_offset = index_of_dim3 - index_of_input_dim3 * scales[3]; - - input_index = index_of_dim0 * input_pitches[0] + - index_of_dim1 * input_pitches[1] + - index_of_input_dim2 * input_pitches[2] + - index_of_input_dim3; - - float x00 = getInputFloat(input_index); - float x10, x01, x11; - - bool end_of_dim2 = false; - if (index_of_input_dim2 == (${inputs[0].dims[2]} - 1)) { - // It's the end in dimension 2 - x01 = x00; - end_of_dim2 = true; - } else { - x01 = getInputFloat(input_index + input_pitches[2]); - } - - if (index_of_input_dim3 == (input_pitches[2] - 1)) { - // It's the end in dimension 3 - x10 = x00; - x11 = x01; - } - else { - x10 = getInputFloat(input_index + 1); - x11 = end_of_dim2 ? x10 : getInputFloat(input_index + input_pitches[2] + 1); - } - - float y0 = x00 + float(y_offset) * (x01 - x00) / float(scales[2]); - float y1 = x10 + float(y_offset) * (x11 - x10) / float(scales[2]); - return y0 + float(x_offset) * (y1 - y0) / float(scales[3]); - }` : - // bilinear 2D - ` - ${getInputFloatFunction} - float process(int indices[2]) { - int input_index = 0; - int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); - - ${precalculatedPitches} - - int m; - int index_of_dim0, index_of_dim1; - index_of_dim0 = output_index / output_pitches[0]; - m = output_index - index_of_dim0 * output_pitches[0]; - index_of_dim1 = m; - - int index_of_input_dim0, index_of_input_dim1, x_offset, y_offset; - index_of_input_dim0 = index_of_dim0 / scales[0]; - y_offset = index_of_dim0 - index_of_input_dim0 * scales[0]; - index_of_input_dim1 = index_of_dim1 / scales[1]; - x_offset = index_of_dim1 - index_of_input_dim1 * scales[1]; - - input_index = index_of_input_dim0 * input_pitches[0] + index_of_input_dim1; - - float x00 = getInputFloat(input_index); - float x10, x01, x11; - - bool end_of_dim0 = false; - if (index_of_input_dim0 == (${inputs[0].dims[0]} - 1)) { - // It's the end in dimension 0 - x01 = x00; - end_of_dim0 = true; - } else { - x01 = getInputFloat(input_index + input_pitches[0]); - } - - if (index_of_input_dim1 == (input_pitches[0] - 1)) { - // It's the end in dimension 1 - x10 = x00; - x11 = x01; - } - else { - x10 = getInputFloat(input_index + 1); - x11 = end_of_dim0 ? x10 : getInputFloat(input_index + input_pitches[0] + 1); - } - - float y0 = x00 + float(y_offset) * (x01 - x00) / float(scales[0]); - float y1 = x10 + float(y_offset) * (x11 - x10) / float(scales[0]); - return y0 + float(x_offset) * (y1 - y0) / float(scales[1]); - }`; return { inputLayouts: [inputLayout], outputLayout, samplers: ['X'], - shaderSource, - variables: [{name: 'scales', type: 'int', arrayLength: this.scales.length}] + shaderSource: ` + ${getInputFloatFunction} +float process(int indices[${dim}]) { + int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); + + ${precalculatedPitches} + + ${ + dim === 2 ? 'int m = output_index; int imageid = 0;' : + `int imageid = output_index / output_pitches[${ + dim - 3}]; int m = output_index - imageid * output_pitches[${dim - 3}];`} + int input_index = imageid * ${inputHeight * inputWidth}; + int output_y = m / output_pitches[${dim - 2}]; + int output_x = m - output_y * output_pitches[${dim - 2}]; + + ${ + extrapolationEnabled ? `if (me[output_y] + me[${outputHeight}+output_x] > 0) { + return float(${extrapolationValue}); + }` : + ''} + + float y_offset_0 = mw[output_y]; + int y_int = mo[output_y]; + float x_offset_0 = mw[${outputHeight}+output_x]; + int x_int = mo[${outputHeight}+output_x]; + input_index += y_int * ${inputWidth} + x_int; + + float x00 = getInputFloat(input_index); + bool end_of_h = (y_int >= ${inputHeight} - 1); + bool end_of_w = (x_int >= ${inputWidth} - 1); + float x10 = end_of_w ? x00 : getInputFloat(input_index + 1); + float x01 = end_of_h ? x00 : getInputFloat(input_index + ${inputWidth}); + float x11 = end_of_w ? x01 : (end_of_h ? x10 : getInputFloat(input_index + ${inputWidth} + 1)); + + float y_offset_1 = 1.0 - y_offset_0; + float x_offset_1 = 1.0 - x_offset_0; + + return x00 * (y_offset_1 * x_offset_1) + + x01 * (y_offset_0 * x_offset_1) + + x10 * (y_offset_1 * x_offset_0) + + x11 * (y_offset_0 * x_offset_0); +}`, + variables }; - } - createRunData(handler: WebGLInferenceHandler, programInfo: ProgramInfo, inputs: Tensor[]): RunData { - const inputTDs = inputs.map((t, i) => handler.getOrCreateTextureData(t, programInfo.inputLayouts[i])); + + } else { // cubic + fillResizeCubicCoordinateMapping( + inputHeight, inputWidth, outputHeight, outputWidth, scalesHeight, scalesWidth, roiStartHeight, roiEndHeight, + roiStartWidth, roiEndWidth, extrapolationEnabled, cubicCoefficientA, excludeOutside, getOriginalCoordinate, + mappingOriginCache, mappingExtrapolateCache, mappingCoeffCache); + const variables: VariableInfo[] = [ + {name: 'mo', type: 'int', arrayLength: mappingOriginCache.length}, + {name: 'mc', type: 'float', arrayLength: mappingCoeffCache.length} + ]; + if (extrapolationEnabled) { + variables.push({name: 'me', type: 'int', arrayLength: mappingExtrapolateCache.length}); + } return { - inputTextureDatas: inputTDs, - outputTextureData: handler.createTextureDataFromLayout(programInfo.outputLayout, inputTDs[0].tensor.type), - uniformData: {scales: this.scales.map(x => Math.ceil(x))} + inputLayouts: [inputLayout], + outputLayout, + samplers: ['X'], + shaderSource: ` + ${getInputFloatFunction} +float rowwise(int x, int y, int offset, float coeff0, float coeff1, float coeff2, float coeff3) { + int row_index = max(0, min(y, ${inputHeight - 1})) * ${inputWidth}; + return coeff0 * getInputFloat(offset + row_index + max(0, min(x - 1, ${inputWidth} - 1))) + + coeff1 * getInputFloat(offset + row_index + max(0, min(x, ${inputWidth} - 1))) + + coeff2 * getInputFloat(offset + row_index + max(0, min(x + 1, ${inputWidth} - 1))) + + coeff3 * getInputFloat(offset + row_index + max(0, min(x + 2, ${inputWidth} - 1))); +} +float process(int indices[${dim}]) { + int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); + + ${precalculatedPitches} + + ${ + dim === 2 ? 'int m = output_index; int imageid = 0;' : + `int imageid = output_index / output_pitches[${ + dim - 3}]; int m = output_index - imageid * output_pitches[${dim - 3}];`} + int input_index = imageid * ${inputHeight * inputWidth}; + int output_y = m / output_pitches[${dim - 2}]; + int output_x = m - output_y * output_pitches[${dim - 2}]; + + ${ + extrapolationEnabled ? `if (me[output_y] + me[${outputHeight}+output_x] > 0) { + return float(${extrapolationValue}); + }` : + ''} + + float w0 = mc[(${outputHeight}+output_x)*4]; + float w1 = mc[(${outputHeight}+output_x)*4+1]; + float w2 = mc[(${outputHeight}+output_x)*4+2]; + float w3 = mc[(${outputHeight}+output_x)*4+3]; + int x_int = mo[${outputHeight}+output_x]; + float y0 = mc[output_y*4]; + float y1 = mc[output_y*4+1]; + float y2 = mc[output_y*4+2]; + float y3 = mc[output_y*4+3]; + int y_int = mo[output_y]; + + return y0 * rowwise(x_int, y_int - 1, input_index, w0, w1, w2, w3) + + y1 * rowwise(x_int, y_int, input_index, w0, w1, w2, w3) + + y2 * rowwise(x_int, y_int + 1, input_index, w0, w1, w2, w3) + + y3 * rowwise(x_int, y_int + 2, input_index, w0, w1, w2, w3); +}`, + variables }; } } + +function fillResizeNearestMapping2D( + inputHeight: number, inputWidth: number, outputHeight: number, outputWidth: number, scalesHeight: number, + scalesWidth: number, roiStartHeight: number, roiEndHeight: number, roiStartWidth: number, roiEndWidth: number, + extrapolationEnabled: boolean, getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc, + getNearestPixel: Upsample.GetNearestPixelFunc, mappingOrigin: number[], mappingExtrapolation: number[]): void { + for (let i = 0; i < outputHeight; i++) { + let dim = i; + const originalCoord = + getOriginalCoordinate(dim, scalesHeight, outputHeight, inputHeight, roiStartHeight, roiEndHeight); + // extrapolate + mappingExtrapolation.push((extrapolationEnabled && (originalCoord < 0 || originalCoord > inputHeight - 1)) ? 1 : 0); + dim = Math.max(0, Math.min(inputHeight - 1, getNearestPixel(originalCoord, scalesHeight < 1))); + // origin + mappingOrigin.push(dim); + } + + for (let i = 0; i < outputWidth; i++) { + let dim = i; + const originalCoord = getOriginalCoordinate(dim, scalesWidth, outputWidth, inputWidth, roiStartWidth, roiEndWidth); + // extrapolate + mappingExtrapolation.push((extrapolationEnabled && (originalCoord < 0 || originalCoord > inputWidth - 1)) ? 1 : 0); + dim = Math.max(0, Math.min(inputWidth - 1, getNearestPixel(originalCoord, scalesWidth < 1))); + // origin + mappingOrigin.push(dim); + } +} + +function fillResizeBilinearCoordinateMapping( + inputHeight: number, inputWidth: number, outputHeight: number, outputWidth: number, scalesHeight: number, + scalesWidth: number, roiStartHeight: number, roiEndHeight: number, roiStartWidth: number, roiEndWidth: number, + extrapolationEnabled: boolean, getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc, mappingOrigin: number[], + mappingWeight: number[], mappingExtrapolation: number[]) { + for (let i = 0; i < outputHeight; i++) { + let inputY = getOriginalCoordinate(i, scalesHeight, outputHeight, inputHeight, roiStartHeight, roiEndHeight); + mappingExtrapolation.push((extrapolationEnabled && (inputY < 0 || inputY > inputHeight - 1)) ? 1 : 0); + inputY = Math.max(0, Math.min(inputY, inputHeight - 1)); + const intY = Math.floor(inputY); + mappingOrigin.push(intY); + mappingWeight.push(intY >= inputHeight - 1 ? 0.5 : inputY - intY); + } + for (let i = 0; i < outputWidth; i++) { + let inputX = getOriginalCoordinate(i, scalesWidth, outputWidth, inputWidth, roiStartWidth, roiEndWidth); + mappingExtrapolation.push((extrapolationEnabled && (inputX < 0 || inputX > inputWidth - 1)) ? 1 : 0); + inputX = Math.max(0, Math.min(inputX, inputWidth - 1)); + const intX = Math.floor(inputX); + mappingOrigin.push(intX); + mappingWeight.push(intX >= inputWidth - 1 ? 0.5 : inputX - intX); + } +} + +function fillResizeCubicCoordinateMapping( + inputHeight: number, inputWidth: number, outputHeight: number, outputWidth: number, scalesHeight: number, + scalesWidth: number, roiStartHeight: number, roiEndHeight: number, roiStartWidth: number, roiEndWidth: number, + extrapolationEnabled: boolean, cubicCoefficientA: number, excludeOutside: boolean, + getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc, mappingOrigin: number[], mappingExtrapolation: number[], + mappingCoeffCache: number[]) { + for (let i = 0; i < outputHeight + outputWidth; i++) { + const isY = i < outputHeight; + const maxInputCoord = isY ? inputHeight : inputWidth; + const inputCoord = getOriginalCoordinate( + isY ? i : i - outputHeight, isY ? scalesHeight : scalesWidth, isY ? outputHeight : outputWidth, maxInputCoord, + isY ? roiStartHeight : roiStartWidth, isY ? roiEndHeight : roiEndWidth); + const intCoord = Math.floor(inputCoord); + const sCoord = Math.abs(intCoord - inputCoord); + let coeffSum = 1.0; + let coeff0 = ((cubicCoefficientA * (sCoord + 1) - 5 * cubicCoefficientA) * (sCoord + 1) + 8 * cubicCoefficientA) * + (sCoord + 1) - + 4 * cubicCoefficientA; + let coeff1 = ((cubicCoefficientA + 2) * sCoord - (cubicCoefficientA + 3)) * sCoord * sCoord + 1; + let coeff2 = ((cubicCoefficientA + 2) * (1 - sCoord) - (cubicCoefficientA + 3)) * (1 - sCoord) * (1 - sCoord) + 1; + let coeff3 = ((cubicCoefficientA * (2 - sCoord) - 5 * cubicCoefficientA) * (2 - sCoord) + 8 * cubicCoefficientA) * + (2 - sCoord) - + 4 * cubicCoefficientA; + if (excludeOutside) { + coeff0 = (intCoord - 1 < 0 || intCoord - 1 >= maxInputCoord) ? 0.0 : coeff0; + coeff1 = (intCoord + 0 < 0 || intCoord + 0 >= maxInputCoord) ? 0.0 : coeff1; + coeff2 = (intCoord + 1 < 0 || intCoord + 1 >= maxInputCoord) ? 0.0 : coeff2; + coeff3 = (intCoord + 2 < 0 || intCoord + 2 >= maxInputCoord) ? 0.0 : coeff3; + coeffSum = coeff0 + coeff1 + coeff2 + coeff3; + } + mappingOrigin.push(intCoord); + mappingExtrapolation.push((extrapolationEnabled && (inputCoord < 0 || inputCoord > maxInputCoord - 1)) ? 1 : 0); + mappingCoeffCache.push(coeff0 / coeffSum); + mappingCoeffCache.push(coeff1 / coeffSum); + mappingCoeffCache.push(coeff2 / coeffSum); + mappingCoeffCache.push(coeff3 / coeffSum); + } +} + +function createUpsampleProgramInfo( + glsl: Glsl, mode: string, dim: number, inputLayout: TextureLayout, outputLayout: TextureLayout, + scales: ReadonlyArray): ProgramInfo { + const outputShape = outputLayout.shape; + const inputShape = inputLayout.shape; + const precalculatedPitches = shaderPrecalculatedPitches(dim, outputShape, inputShape); + const getInputFloatFunction = shaderGetInputFloatFunction(inputLayout, glsl); + + const shaderSource = mode === 'nearest' ? + // nearest + ` + ${getInputFloatFunction} + float process(int indices[${dim}]) { + int input_index = 0; + int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); + + ${precalculatedPitches} + + int d, m; + for (int dim = 0; dim < ${dim}; ++dim) { + d = output_index / output_pitches[dim]; + m = output_index - d * output_pitches[dim]; + output_index = m; + + if (scales[dim] != 1 && d > 0) { + int d2 = d / scales[dim]; + m = d - d2 * scales[dim]; + d = d2; + } + input_index += input_pitches[dim] * d; + } + + return getInputFloat(input_index); + }` : + dim === 4 ? + // bilinear 4D + ` + ${getInputFloatFunction} + float process(int indices[4]) { + int input_index = 0; + int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); + + ${precalculatedPitches} + + int m; + int index_of_dim0, index_of_dim1, index_of_dim2, index_of_dim3; + index_of_dim0 = output_index / output_pitches[0]; + m = output_index - index_of_dim0 * output_pitches[0]; + index_of_dim1 = m / output_pitches[1]; + m = m - index_of_dim1 * output_pitches[1]; + index_of_dim2 = m / output_pitches[2]; + m = m - index_of_dim2 * output_pitches[2]; + index_of_dim3 = m; + + int index_of_input_dim2, index_of_input_dim3, x_offset, y_offset; + index_of_input_dim2 = index_of_dim2 / scales[2]; + y_offset = index_of_dim2 - index_of_input_dim2 * scales[2]; + index_of_input_dim3 = index_of_dim3 / scales[3]; + x_offset = index_of_dim3 - index_of_input_dim3 * scales[3]; + + input_index = index_of_dim0 * input_pitches[0] + + index_of_dim1 * input_pitches[1] + + index_of_input_dim2 * input_pitches[2] + + index_of_input_dim3; + + float x00 = getInputFloat(input_index); + float x10, x01, x11; + + bool end_of_dim2 = false; + if (index_of_input_dim2 == (${inputShape[2]} - 1)) { + // It's the end in dimension 2 + x01 = x00; + end_of_dim2 = true; + } else { + x01 = getInputFloat(input_index + input_pitches[2]); + } + + if (index_of_input_dim3 == (input_pitches[2] - 1)) { + // It's the end in dimension 3 + x10 = x00; + x11 = x01; + } + else { + x10 = getInputFloat(input_index + 1); + x11 = end_of_dim2 ? x10 : getInputFloat(input_index + input_pitches[2] + 1); + } + + float y0 = x00 + float(y_offset) * (x01 - x00) / float(scales[2]); + float y1 = x10 + float(y_offset) * (x11 - x10) / float(scales[2]); + return y0 + float(x_offset) * (y1 - y0) / float(scales[3]); + }` : + // bilinear 2D + ` + ${getInputFloatFunction} + float process(int indices[2]) { + int input_index = 0; + int output_index = coordsToOffset(TexCoords, ${outputLayout.width}, ${outputLayout.height}); + + ${precalculatedPitches} + + int m; + int index_of_dim0, index_of_dim1; + index_of_dim0 = output_index / output_pitches[0]; + m = output_index - index_of_dim0 * output_pitches[0]; + index_of_dim1 = m; + + int index_of_input_dim0, index_of_input_dim1, x_offset, y_offset; + index_of_input_dim0 = index_of_dim0 / scales[0]; + y_offset = index_of_dim0 - index_of_input_dim0 * scales[0]; + index_of_input_dim1 = index_of_dim1 / scales[1]; + x_offset = index_of_dim1 - index_of_input_dim1 * scales[1]; + + input_index = index_of_input_dim0 * input_pitches[0] + index_of_input_dim1; + + float x00 = getInputFloat(input_index); + float x10, x01, x11; + + bool end_of_dim0 = false; + if (index_of_input_dim0 == (${inputShape[0]} - 1)) { + // It's the end in dimension 0 + x01 = x00; + end_of_dim0 = true; + } else { + x01 = getInputFloat(input_index + input_pitches[0]); + } + + if (index_of_input_dim1 == (input_pitches[0] - 1)) { + // It's the end in dimension 1 + x10 = x00; + x11 = x01; + } + else { + x10 = getInputFloat(input_index + 1); + x11 = end_of_dim0 ? x10 : getInputFloat(input_index + input_pitches[0] + 1); + } + + float y0 = x00 + float(y_offset) * (x01 - x00) / float(scales[0]); + float y1 = x10 + float(y_offset) * (x11 - x10) / float(scales[0]); + return y0 + float(x_offset) * (y1 - y0) / float(scales[1]); + }`; + return { + inputLayouts: [inputLayout], + outputLayout, + samplers: ['X'], + shaderSource, + variables: [{name: 'scales', type: 'int', arrayLength: scales.length}] + }; +} + +function shaderPrecalculatedPitches( + dim: number, outputShape: ReadonlyArray, inputShape: ReadonlyArray) { + const outputPitches = new Array(dim); + const inputPitches = new Array(dim); + let precalculatedPitches = ` + int output_pitches[${dim}]; + int input_pitches[${dim}]; + `; + for (let d = dim - 1; d >= 0; d--) { + outputPitches[d] = (d === dim - 1) ? 1 : outputPitches[d + 1] * outputShape[d + 1]; + inputPitches[d] = (d === dim - 1) ? 1 : inputPitches[d + 1] * inputShape[d + 1]; + + precalculatedPitches += ` + output_pitches[${d}] = ${outputPitches[d]}; + input_pitches[${d}] = ${inputPitches[d]}; + `; + } + return precalculatedPitches; +} + +function shaderGetInputFloatFunction(inputLayout: TextureLayout, glsl: Glsl) { + return ` +float getInputFloat(int index) { + vec2 coords = offsetToCoords(index, ${inputLayout.width}, ${inputLayout.height}); + float value = getColorAsFloat(${glsl.texture2D}(X, coords)); + return value; +} +`; +} diff --git a/lib/ops/cast.ts b/lib/ops/cast.ts index 6b0ff3b3..ad6f857d 100644 --- a/lib/ops/cast.ts +++ b/lib/ops/cast.ts @@ -3,7 +3,7 @@ import {Attribute} from '../attribute'; import {InferenceHandler} from '../backend'; -import {NUMBER_TYPES, Operator} from '../operators'; +import {Operator} from '../operators'; import {Tensor} from '../tensor'; import {ProtoUtil} from '../util'; @@ -25,7 +25,7 @@ export abstract class Cast implements Operator { } protected checkInputTypes(inputs: Tensor[]): boolean { - if (NUMBER_TYPES.indexOf(inputs[0].type) === -1) { + if (inputs[0].type === 'string') { return false; } return true; diff --git a/lib/ops/reshape.ts b/lib/ops/reshape.ts index bcedf4f3..1b0f6d2b 100644 --- a/lib/ops/reshape.ts +++ b/lib/ops/reshape.ts @@ -3,7 +3,7 @@ import {Attribute} from '../attribute'; import {InferenceHandler} from '../backend'; -import {Operator} from '../operators'; +import {NUMBER_TYPES, Operator} from '../operators'; import {Tensor} from '../tensor'; export abstract class Reshape implements Operator { @@ -20,7 +20,7 @@ export abstract class Reshape implements Operator { } protected checkInputTypes(inputs: Tensor[]): boolean { - if (inputs[0].type !== 'float32' && inputs[0].type !== 'float64') { + if (NUMBER_TYPES.indexOf(inputs[0].type) === -1) { return false; } diff --git a/lib/ops/upsample.ts b/lib/ops/upsample.ts index 29567665..b75176fa 100644 --- a/lib/ops/upsample.ts +++ b/lib/ops/upsample.ts @@ -3,33 +3,104 @@ import {Attribute} from '../attribute'; import {InferenceHandler} from '../backend'; +import {Graph} from '../graph'; import {Operator} from '../operators'; import {Tensor} from '../tensor'; +export declare namespace Upsample { + interface GetNearestPixelFunc { + (a: number, b: boolean): number; + } + interface GetOriginalCoordinateFunc { + (xResized: number, xScale: number, lengthResized: number, lengthOriginal: number, roiStart: number, + roiEnd: number): number; + } +} + export abstract class Upsample implements Operator { + constructor(protected opset: number) {} + abstract run(inferenceHandler: InferenceHandler, inputs: Tensor[]): Tensor[]|Promise; - initialize(attributes: Attribute): void { - this.mode = attributes.getString('mode', 'nearest'); - this.scales = attributes.getFloats('scales'); + initialize(attributes: Attribute, node: Graph.Node, graph: Graph): void { + this.isResize = (this.opset >= 10); - if (this.mode !== 'nearest' && this.mode !== 'linear') { + this.mode = attributes.getString('mode', 'nearest'); + if (this.mode !== 'nearest' && this.mode !== 'linear' && (this.opset < 11 || this.mode !== 'cubic')) { throw new Error(`unrecognized mode: ${this.mode}`); } - if (this.mode === 'linear' && this.scales.length !== 2 && this.scales.length !== 4) { - throw new Error(`only support 2-D or 4-D upsampling for linear mode`); + if (this.opset < 9) { + this.scales = attributes.getFloats('scales'); + scalesValidataion(this.scales, this.mode, this.isResize); + } + + this.extrapolationValue = attributes.getFloat('extrapolation_value', 0.0); + + this.coordinateTransformMode = + this.opset > 10 ? attributes.getString('coordinate_transformation_mode', 'half_pixel') : 'asymmetric'; + if ([ + 'asymmetric', 'pytorch_half_pixel', 'tf_half_pixel_for_nn', 'align_corners', 'tf_crop_and_resize', + 'half_pixel' + ].indexOf(this.coordinateTransformMode) === -1) { + throw new Error(`coordinate_transform_mode '${this.coordinateTransformMode}' is not supported`); + } + this.useExtrapolation = this.needRoiInput = (this.coordinateTransformMode === 'tf_crop_and_resize'); + + this.nearestMode = + (this.mode === 'nearest' && this.opset >= 11) ? attributes.getString('nearest_mode', 'round_prefer_floor') : ''; + if (['round_prefer_floor', 'round_prefer_ceil', 'floor', 'ceil', ''].indexOf(this.nearestMode) === -1) { + throw new Error(`nearest_mode '${this.nearestMode}' is not supported`); + } + + this.cubicCoefficientA = attributes.getFloat('cubic_coeff_a', -0.75); + this.excludeOutside = attributes.getInt('exclude_outside', 0) !== 0; + if (this.excludeOutside && this.mode !== 'cubic') { + throw new Error('exclude_outside can be set to 1 only when mode is CUBIC.'); + } + + this.useNearest2xOptimization = (this.opset < 11) ? + true : + (this.mode === 'nearest' && this.coordinateTransformMode === 'asymmetric' && this.nearestMode === 'floor'); + + if (this.opset > 10) { + this.roiInputIdx = 1; + this.scalesInputIdx = 2; + this.sizesInputIdx = 3; + } else if (this.opset === 9 || this.opset === 10) { + this.scalesInputIdx = 1; + } + + if (this.scalesInputIdx > 0) { + const scale = graph.getValues()[node.inputs[this.scalesInputIdx]].tensor; + + if (scale && scale.dims.length > 0) { + this.scales = Array.from(scale.floatData); + scalesValidataion(this.scales, this.mode, this.isResize); + } + } + + // roi is only needed when coordinate transformation mode is tf_crop_and_resize + // for all other modes no need to read roi input + if (this.roiInputIdx > 0 && this.needRoiInput) { + const roi = graph.getValues()[node.inputs[this.roiInputIdx]].tensor; + if (roi) { + this.roi = Array.from(roi.floatData); + } } - this.roi = new Array(this.scales.length * 2).fill(0); + this.getOriginalCoordinate = getOriginalCoordinateFromResizedCoordinate(this.coordinateTransformMode); + this.getNearestPixel = getNearestPixelFromOriginal(this.nearestMode); } checkInputs(inputs: Tensor[]): boolean { - if (!inputs || inputs.length !== 1) { + if (!inputs || (this.opset < 9 && inputs.length !== 1) || + (this.opset >= 9 && this.opset < 11 && inputs.length !== 2) || + (this.opset >= 11 && inputs.length !== 3 && inputs.length !== 4)) { return false; } - if (inputs[0].dims.length !== this.scales.length) { + if (this.scales && inputs[0].dims.length !== this.scales.length) { return false; } @@ -44,37 +115,162 @@ export abstract class Upsample implements Operator { return true; } + protected prepare(inputs: Tensor[]): [number[], number[], ReadonlyArray] { + const x = inputs[0]; + const xDims = x.dims; + + // get roi data + let roi = this.roi; + if (!roi) { + if (this.needRoiInput) { + if (this.roiInputIdx <= 0) { + throw new Error('Invalid roi input index.'); + } + roi = parseRoiData(inputs[this.roiInputIdx]); + } else { + roi = new Array(xDims.length * 2).fill(0); + } + } + + let scales = this.scales; + let outputSizes: number[]|undefined; + if (!scales) { + const scalesTensor = inputs[this.scalesInputIdx]; + if (scalesTensor && scalesTensor.size !== 0) { + if (inputs[this.sizesInputIdx]) { + throw new Error('Only one of scales or sizes must be provided as input.'); + } + scales = parseScalesData(scalesTensor, this.mode, this.isResize); + } else { + const sizesTensor = inputs[this.sizesInputIdx]; + if (!sizesTensor || sizesTensor.size === 0) { + throw new Error('Either scales or sizes MUST be provided as input.'); + } + + outputSizes = Array.from(sizesTensor.integerData); + scales = parseScalesDataFromOutputSize(outputSizes, xDims, this.mode, this.isResize); + } + } else { + if (inputs[this.sizesInputIdx]) { + throw new Error('Only one of scales or sizes must be provided as input.'); + } + } + + const yDims = outputSizes || computeOutputShape(scales, xDims); + + return [roi, scales, yDims]; + } + + protected isResize: boolean; protected mode: string; protected scales: number[]; + protected extrapolationValue: number; + protected coordinateTransformMode: string; + protected useExtrapolation: boolean; + protected needRoiInput: boolean; + protected nearestMode: string; + protected cubicCoefficientA: number; + protected excludeOutside: boolean; + protected useNearest2xOptimization: boolean; + protected roiInputIdx: number; + protected scalesInputIdx: number; + protected sizesInputIdx: number; protected roi: number[]; -} - -export abstract class UpsampleV9 implements Operator { - abstract run(inferenceHandler: InferenceHandler, inputs: Tensor[]): Tensor[]|Promise; - initialize(attributes: Attribute): void { - this.mode = attributes.getString('mode', 'nearest'); + protected getOriginalCoordinate: Upsample.GetOriginalCoordinateFunc; + protected getNearestPixel: Upsample.GetNearestPixelFunc; +} - if (this.mode !== 'nearest' && this.mode !== 'linear') { - throw new Error(`unrecognized mode: ${this.mode}`); +function scalesValidataion(scales: number[], mode: string, isResize: boolean) { + if (!isResize) { + for (const scale of scales) { + if (scale < 1) { + throw new Error('Scale value should be greater than or equal to 1.'); + } + } + } else { + for (const scale of scales) { + if (scale <= 0) { + throw new Error('Scale value should be greater than 0.'); + } } } - - checkInputs(inputs: Tensor[]): boolean { - if (!inputs || inputs.length !== 2) { - return false; + if (mode === 'linear' || mode === 'cubic') { + if (scales.length !== 2 && (scales.length !== 4 || scales[0] !== 1 || scales[1] !== 1)) { + throw new Error(`'Linear' mode and 'Cubic' mode only support 2-D inputs ('Bilinear', 'Bicubic') or 4-D inputs\ +with the corresponding outermost 2 scale values being 1 in the ${isResize ? 'Resize' : 'Upsample'} opeartor.`); } - - return this.checkInputTypes(inputs); } +} - protected checkInputTypes(inputs: Tensor[]): boolean { - if (inputs[0].type === 'string') { - return false; +export function parseRoiData(roi: Tensor): number[] { + return roi.size > 0 ? Array.from(roi.floatData) : []; +} + +export function parseScalesData(scale: Tensor, mode: string, isResize: boolean): number[] { + const scales = Array.from(scale.floatData); + scalesValidataion(scales, mode, isResize); + return scales; +} + +export function parseScalesDataFromOutputSize( + yDims: ReadonlyArray, xDims: ReadonlyArray, mode: string, isResize: boolean): number[] { + const length = xDims.length; + const scales = new Array(length); + + for (let i = 0, end = length; i < end; i++) { + if (xDims[i] === 0) { + if (yDims[i] !== 0) { + throw new Error('Input dim is zero but required output dim is non-zero.'); + } + scales[i] = 1; + } else { + scales[i] = yDims[i] / xDims[i]; } + } + scalesValidataion(scales, mode, isResize); + return scales; +} - return true; +export function computeOutputShape(scales: ReadonlyArray, inputDims: ReadonlyArray): number[] { + return inputDims.map((dim, i) => Math.floor(dim * scales[i])); +} + +function getOriginalCoordinateFromResizedCoordinate(mode: string): Upsample.GetOriginalCoordinateFunc { + switch (mode) { + case 'asymmetric': + return (xResized: number, xScale: number) => xResized / xScale; + case 'pytorch_half_pixel': + return (xResized: number, xScale: number, lengthResized: number) => + lengthResized > 1 ? (xResized + 0.5) / xScale - 0.5 : 0.0; + case 'tf_half_pixel_for_nn': + return (xResized: number, xScale: number) => (xResized + 0.5) / xScale; + case 'align_corners': + return (xResized: number, xScale: number, lengthResized: number, lengthOriginal: number) => + lengthResized === 1 ? 0 : xResized * (lengthOriginal - 1) / (lengthResized - 1); + case 'tf_crop_and_resize': + return (xResized: number, xScale: number, lengthResized: number, lengthOriginal: number, roiStart: number, + roiEnd: number) => + (lengthResized > 1 ? roiStart * (lengthOriginal - 1) + + (xResized * (roiEnd - roiStart) * (lengthOriginal - 1)) / (lengthResized - 1) : + 0.5 * (roiStart + roiEnd) * (lengthOriginal - 1)); + default: //'half_pixel' + return (xResized: number, xScale: number) => (xResized + 0.5) / xScale - 0.5; } +} - protected mode: string; +function getNearestPixelFromOriginal(mode: string): Upsample.GetNearestPixelFunc { + switch (mode) { + case '': + return (xOriginal: number, isDownSample: boolean) => isDownSample ? Math.ceil(xOriginal) : Math.floor(xOriginal); + case 'round_prefer_ceil': + return (xOriginal: number) => Math.round(xOriginal); + case 'floor': + return (xOriginal: number) => Math.floor(xOriginal); + case 'ceil': + return (xOriginal: number) => Math.ceil(xOriginal); + default: // round_prefer_floor + return (xOriginal: number) => + xOriginal === Math.floor(xOriginal) + 0.5 ? Math.floor(xOriginal) : Math.round(xOriginal); + } } diff --git a/lib/wasm-binding-core.ts b/lib/wasm-binding-core.ts index 457538c5..c1924cd4 100644 --- a/lib/wasm-binding-core.ts +++ b/lib/wasm-binding-core.ts @@ -267,6 +267,7 @@ export class WasmBinding { // retrieve data parameters (in/inout) from emscripten heap after ccall() static ccallDeserialize(buffer: Uint8Array, offset: number[], params: WasmCallArgument[]) { const heapF32 = new Float32Array(buffer.buffer, buffer.byteOffset); + const heapI32 = new Int32Array(buffer.buffer, buffer.byteOffset); const heapU8 = new Uint8Array(buffer.buffer, buffer.byteOffset); for (let i = 0; i < params.length; i++) { @@ -288,6 +289,10 @@ export class WasmBinding { const float32Array = (paramData as Float32Array); float32Array.set(heapF32.subarray(offset32, offset32 + float32Array.length)); break; + case 'int32ptr': + const int32Array = (paramData as Int32Array); + int32Array.set(heapI32.subarray(offset32, offset32 + int32Array.length)); + break; case 'boolptr': const boolArray = (paramData as Uint8Array); boolArray.set(heapU8.subarray(offset8, offset8 + boolArray.length)); diff --git a/package-lock.json b/package-lock.json index 0fe6c361..7a603661 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3559,9 +3559,9 @@ "dev": true }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==", "dev": true }, "inline-source-map": { diff --git a/src/wasm-build-config.json b/src/wasm-build-config.json index 95a99ee8..0ab6fb63 100644 --- a/src/wasm-build-config.json +++ b/src/wasm-build-config.json @@ -10,6 +10,9 @@ "_mul_f32", "_div_f32", "_prelu_f32", + "_add_i32", + "_sub_i32", + "_mul_i32", "_xor_u8", "_or_u8", "_and_u8", diff --git a/src/wasm-ops/binary-op.cpp b/src/wasm-ops/binary-op.cpp index e9d588fb..c8c3e3c6 100644 --- a/src/wasm-ops/binary-op.cpp +++ b/src/wasm-ops/binary-op.cpp @@ -40,6 +40,29 @@ void prelu_f32(void *data) { float *output = PARAM_FLOAT_PTR(data, dataIndex[7]); binary_imp(data, input_1, input_2, output); } + +void add_i32(void *data) { + uint32_t *dataIndex = static_cast(data); + const int32_t *input_1 = PARAM_INT32_PTR(data, dataIndex[1]); + const int32_t *input_2 = PARAM_INT32_PTR(data, dataIndex[4]); + int32_t *output = PARAM_INT32_PTR(data, dataIndex[7]); + binary_imp(data, input_1, input_2, output); +} +void sub_i32(void *data) { + uint32_t *dataIndex = static_cast(data); + const int32_t *input_1 = PARAM_INT32_PTR(data, dataIndex[1]); + const int32_t *input_2 = PARAM_INT32_PTR(data, dataIndex[4]); + int32_t *output = PARAM_INT32_PTR(data, dataIndex[7]); + binary_imp(data, input_1, input_2, output); +} +void mul_i32(void *data) { + uint32_t *dataIndex = static_cast(data); + const int32_t *input_1 = PARAM_INT32_PTR(data, dataIndex[1]); + const int32_t *input_2 = PARAM_INT32_PTR(data, dataIndex[4]); + int32_t *output = PARAM_INT32_PTR(data, dataIndex[7]); + binary_imp(data, input_1, input_2, output); +} + void xor_u8(void *data) { uint32_t *dataIndex = static_cast(data); const uint8_t *input_1 = PARAM_BOOL_PTR(data, dataIndex[1]); diff --git a/src/wasm-ops/binary-op.h b/src/wasm-ops/binary-op.h index 2f38b61b..b42f40e6 100644 --- a/src/wasm-ops/binary-op.h +++ b/src/wasm-ops/binary-op.h @@ -17,6 +17,10 @@ void mul_f32(void *); void div_f32(void *); void prelu_f32(void *); +void add_i32(void *); +void sub_i32(void *); +void mul_i32(void *); + // Logical ops void xor_u8(void *); void or_u8(void *); diff --git a/test/data/ops/add_int32.jsonc b/test/data/ops/add_int32.jsonc new file mode 100644 index 00000000..b87a6857 --- /dev/null +++ b/test/data/ops/add_int32.jsonc @@ -0,0 +1,31 @@ +[ + { + "name": "Add with no attributes", + "operator": "Add", + "attributes": [], + "cases": [ + { + "name": "T[2,4] T[2,4] (int32)", + "inputs": [ + { + "data": [1, 2, 1, 3, 2, 3, 1, 2], + "dims": [2, 4], + "type": "int32" + }, + { + "data": [2, 1, 1, 2, 2, 3, 1, 4], + "dims": [2, 4], + "type": "int32" + } + ], + "outputs": [ + { + "data": [3, 3, 2, 5, 4, 6, 2, 6], + "dims": [2, 4], + "type": "int32" + } + ] + } + ] + } +] diff --git a/test/data/ops/gather.jsonc b/test/data/ops/gather.jsonc new file mode 100644 index 00000000..3b1b0e38 --- /dev/null +++ b/test/data/ops/gather.jsonc @@ -0,0 +1,97 @@ +[ + { + "name": "Gather", + "operator": "Gather", + "attributes": [], + "cases": [ + { + "name": "data[4] indices[]", + "inputs": [ + { + "data": [1, 2, 3, 4], + "dims": [4], + "type": "float32" + }, + { + "data": [1], + "dims": [], + "type": "int32" + } + ], + "outputs": [ + { + "data": [2], + "dims": [], + "type": "float32" + } + ] + }, + { + "name": "data[4] indices[1]", + "inputs": [ + { + "data": [1, 2, 3, 4], + "dims": [4], + "type": "float32" + }, + { + "data": [1], + "dims": [1], + "type": "int32" + } + ], + "outputs": [ + { + "data": [2], + "dims": [1], + "type": "float32" + } + ] + }, + { + "name": "data[2,4] indices[]", + "inputs": [ + { + "data": [1, 2, 3, 4, 5, 6, 7, 8], + "dims": [2, 4], + "type": "float32" + }, + { + "data": [1], + "dims": [], + "type": "int32" + } + ], + "outputs": [ + { + "data": [5, 6, 7, 8], + "dims": [4], + "type": "float32" + } + ] + }, + { + "name": "data[2,4] indices[1]", + "inputs": [ + { + "data": [1, 2, 3, 4, 5, 6, 7, 8], + "dims": [2, 4], + "type": "float32" + }, + { + "data": [1], + "dims": [1], + "type": "int32" + } + ], + "outputs": [ + { + "data": [5, 6, 7, 8], + "dims": [1, 4], + "type": "float32" + } + ] + } + ] + } +] diff --git a/test/data/ops/mul_int32.jsonc b/test/data/ops/mul_int32.jsonc new file mode 100644 index 00000000..ddc0032f --- /dev/null +++ b/test/data/ops/mul_int32.jsonc @@ -0,0 +1,31 @@ +[ + { + "name": "Mul with no attributes", + "operator": "Mul", + "attributes": [], + "cases": [ + { + "name": "T[2,4] T[2,4] (int32)", + "inputs": [ + { + "data": [1, 2, 1, 3, 2, 3, 1, 2], + "dims": [2, 4], + "type": "int32" + }, + { + "data": [2, 1, 1, 2, 2, 3, 1, 4], + "dims": [2, 4], + "type": "int32" + } + ], + "outputs": [ + { + "data": [2, 2, 1, 6, 4, 9, 1, 8], + "dims": [2, 4], + "type": "int32" + } + ] + } + ] + } +] diff --git a/test/data/ops/sub_int32.jsonc b/test/data/ops/sub_int32.jsonc new file mode 100644 index 00000000..0ed2bc12 --- /dev/null +++ b/test/data/ops/sub_int32.jsonc @@ -0,0 +1,31 @@ +[ + { + "name": "Sub with no attributes", + "operator": "Sub", + "attributes": [], + "cases": [ + { + "name": "T[2,4] T[2,4] (int32)", + "inputs": [ + { + "data": [1, 2, 1, 3, 2, 3, 1, 2], + "dims": [2, 4], + "type": "int32" + }, + { + "data": [2, 1, 1, 2, 2, 3, 1, 4], + "dims": [2, 4], + "type": "int32" + } + ], + "outputs": [ + { + "data": [-1, 1, 0, 1, 0, 0, 0, -2], + "dims": [2, 4], + "type": "int32" + } + ] + } + ] + } +] diff --git a/test/test-suite-whitelist.jsonc b/test/test-suite-whitelist.jsonc index 9a1d2d63..1abc0e7f 100644 --- a/test/test-suite-whitelist.jsonc +++ b/test/test-suite-whitelist.jsonc @@ -176,6 +176,30 @@ "test_reshape_one_dim", "test_reshape_reduced_dims", "test_reshape_reordered_dims", + "test_resize_downsample_scales_cubic", + "test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", + //"test_resize_downsample_scales_cubic_align_corners", // results mismatch with onnx tests + "test_resize_downsample_scales_linear", + //"test_resize_downsample_scales_linear_align_corners", // results mismatch with onnx tests + "test_resize_downsample_scales_nearest", + "test_resize_downsample_sizes_cubic", + "test_resize_downsample_sizes_linear_pytorch_half_pixel", + "test_resize_downsample_sizes_nearest", + "test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", + "test_resize_tf_crop_and_resize", + //"test_resize_tf_crop_and_resize_extrapolation_value", + "test_resize_upsample_scales_cubic", + "test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", + "test_resize_upsample_scales_cubic_align_corners", + "test_resize_upsample_scales_cubic_asymmetric", + "test_resize_upsample_scales_linear", + "test_resize_upsample_scales_linear_align_corners", + "test_resize_upsample_scales_nearest", + "test_resize_upsample_sizes_cubic", + "test_resize_upsample_sizes_nearest", + "v12/test_resize_upsample_sizes_nearest_ceil_half_pixel", + "v12/test_resize_upsample_sizes_nearest_floor_align_corners", + "v12/test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", "v{7,8,9}/test_slice", "v{7,8,9}/test_slice_default_axes", "v{7,8,9}/test_slice_end_out_of_bounds", @@ -231,6 +255,7 @@ "abs.jsonc", "acos.jsonc", "add.jsonc", + "add_int32.jsonc", "and.jsonc", "asin.jsonc", "ceil.jsonc", @@ -247,6 +272,7 @@ "log.jsonc", "matmul.jsonc", "mul.jsonc", + "mul_int32.jsonc", "neg.jsonc", //"not.jsonc", "or.jsonc", @@ -262,6 +288,7 @@ //"split.jsonc", "sqrt.jsonc", "sub.jsonc", + "sub_int32.jsonc", "softmax.jsonc", "tan.jsonc", "transpose.jsonc", @@ -452,6 +479,93 @@ "test_reduce_sum_square_do_not_keepdims_random", "test_reduce_sum_square_keepdims_example", "test_reduce_sum_square_keepdims_random", + { + "name": "test_resize_downsample_scales_cubic", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", + "condition": "^((?!Safari).)*$" + }, + //"test_resize_downsample_scales_cubic_align_corners", // results mismatch with onnx tests + { + "name": "test_resize_downsample_scales_linear", + "condition": "^((?!Safari).)*$" + }, + //"test_resize_downsample_scales_linear_align_corners", // results mismatch with onnx tests + { + "name": "test_resize_downsample_scales_nearest", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_downsample_sizes_cubic", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_downsample_sizes_linear_pytorch_half_pixel", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_downsample_sizes_nearest", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_tf_crop_and_resize", + "condition": "^((?!Safari).)*$" + }, + //"test_resize_tf_crop_and_resize_extrapolation_value", + { + "name": "test_resize_upsample_scales_cubic", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_scales_cubic_align_corners", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_scales_cubic_asymmetric", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_scales_linear", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_scales_linear_align_corners", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_scales_nearest", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_sizes_cubic", + "condition": "^((?!Safari).)*$" + }, + { + "name": "test_resize_upsample_sizes_nearest", + "condition": "^((?!Safari).)*$" + }, + { + "name": "v12/test_resize_upsample_sizes_nearest_ceil_half_pixel", + "condition": "^((?!Safari).)*$" + }, + { + "name": "v12/test_resize_upsample_sizes_nearest_floor_align_corners", + "condition": "^((?!Safari).)*$" + }, + { + "name": "v12/test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", + "condition": "^((?!Safari).)*$" + }, "test_split_variable_parts_default_axis", "test_split_variable_parts_1d", "test_split_variable_parts_2d", @@ -478,6 +592,7 @@ "test_transpose_all_permutations_5", "test_transpose_default", "test_unsqueeze", + "v{7,8,9}/test_upsample_nearest", "test_xor_bcast3v1d", "test_xor_bcast3v2d", "test_xor_bcast4v2d", @@ -633,8 +748,11 @@ "conv.jsonc", "softmax.jsonc", "add.jsonc", + "add_int32.jsonc", "sub.jsonc", + "sub_int32.jsonc", "mul.jsonc", + "mul_int32.jsonc", "div.jsonc", "and.jsonc", "or.jsonc",