Skip to content

Commit

Permalink
Improve model-building helpers to accept inputs of a potentially diff…
Browse files Browse the repository at this point in the history 
…erent type (e.g., int64 shape/indices/etc)
  • Loading branch information
@adrianlizarraga
adrianlizarraga committed Sep 19, 2023
1 parent 1d205bb commit 80c8fb4
Show file tree
Hide file tree
Showing 15 changed files with 114 additions and 266 deletions.
6 changes: 3 additions & 3 deletions onnxruntime/test/providers/qnn/average_pool_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ static void RunAveragePoolOpTest(const std::string& op_type,
provider_options["backend_path"] = "libQnnCpu.so";
#endif

RunQnnModelTest(BuildOpTestCase(op_type, input_defs, attrs),
RunQnnModelTest(BuildOpTestCase<float>(op_type, input_defs, {}, attrs),
provider_options,
opset,
expected_ep_assignment);
Expand All @@ -53,8 +53,8 @@ static void RunQDQAveragePoolOpTest(const std::string& op_type,
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy(BuildOpTestCase(op_type, input_defs, attrs),
BuildQDQOpTestCase<QuantType>(op_type, input_defs, attrs),
TestQDQModelAccuracy(BuildOpTestCase<float>(op_type, input_defs, {}, attrs),
BuildQDQOpTestCase<QuantType>(op_type, input_defs, {}, attrs),
provider_options,
opset,
expected_ep_assignment);
Expand Down
6 changes: 3 additions & 3 deletions onnxruntime/test/providers/qnn/clip_op_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ static void RunClipTestOnCPU(const std::vector<TestInputDef<DataType>>& input_de
provider_options["backend_path"] = "libQnnCpu.so";
#endif

RunQnnModelTest(BuildOpTestCase<DataType>("Clip", input_defs, {}),
RunQnnModelTest(BuildOpTestCase<DataType>("Clip", input_defs, {}, {}),
provider_options,
opset,
expected_ep_assignment);
Expand Down Expand Up @@ -116,8 +116,8 @@ static void RunQDQClipTestOnHTP(const std::vector<TestInputDef<float>>& input_de
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy(BuildOpTestCase("Clip", input_defs, {}), // baseline float32 model
BuildQDQClipTestCase<QType>(input_defs), // QDQ model
TestQDQModelAccuracy(BuildOpTestCase<float>("Clip", input_defs, {}, {}), // baseline float32 model
BuildQDQClipTestCase<QType>(input_defs), // QDQ model
provider_options,
opset,
expected_ep_assignment);
Expand Down
8 changes: 4 additions & 4 deletions onnxruntime/test/providers/qnn/flatten_op_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ static void RunFlattenTestOnCPU(const TestInputDef<DataType>& input_def,
provider_options["backend_path"] = "libQnnCpu.so";
#endif

RunQnnModelTest(BuildOpTestCase<DataType>("Flatten", {input_def}, attrs),
RunQnnModelTest(BuildOpTestCase<DataType>("Flatten", {input_def}, {}, attrs),
provider_options,
opset,
expected_ep_assignment);
Expand All @@ -50,7 +50,7 @@ static void RunFlattenTestOnHTP(const TestInputDef<DataType>& input_def,
provider_options["backend_path"] = "libQnnHtp.so";
#endif

RunQnnModelTest(BuildOpTestCase<DataType>("Flatten", {input_def}, attrs),
RunQnnModelTest(BuildOpTestCase<DataType>("Flatten", {input_def}, {}, attrs),
provider_options,
opset,
expected_ep_assignment);
Expand All @@ -71,8 +71,8 @@ static void RunQDQFlattenTestOnHTP(const TestInputDef<float>& input_def,
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy(BuildOpTestCase<float>("Flatten", {input_def}, attrs), // baseline float32 model
BuildQDQOpTestCase<QType>("Flatten", {input_def}, attrs), // QDQ model
TestQDQModelAccuracy(BuildOpTestCase<float>("Flatten", {input_def}, {}, attrs), // baseline float32 model
BuildQDQOpTestCase<QType>("Flatten", {input_def}, {}, attrs), // QDQ model
provider_options,
opset,
expected_ep_assignment);
Expand Down
64 changes: 17 additions & 47 deletions onnxruntime/test/providers/qnn/gather_op_htp_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#include <string>
#include "core/graph/graph.h"
#include "core/graph/node_attr_utils.h"

#include "test/providers/qnn/qnn_test_utils.h"

Expand All @@ -14,60 +15,29 @@ namespace onnxruntime {
namespace test {
#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)

// Function that builds a float model with a Gather op.
template <typename IndicesType = int32_t>
static GetTestModelFn BuildGatherOpTestCase(const TestInputDef<float>& input_def,
const TestInputDef<IndicesType>& indices_def,
int64_t axis = 0) {
return [input_def, indices_def, axis](ModelTestBuilder& builder) {
NodeArg* input = MakeTestInput(builder, input_def);
NodeArg* indices = MakeTestInput(builder, indices_def);
NodeArg* output = builder.MakeOutput();

Node& gather_node = builder.AddNode("Gather", {input, indices}, {output});
gather_node.AddAttribute("axis", axis);
};
}

// Function that builds a QDQ model with a Gather op.
template <typename QuantType = uint8_t, typename IndicesType = int32_t>
static GetTestQDQModelFn<QuantType> BuildQDQGatherOpTestCase(const TestInputDef<float>& input_def,
const TestInputDef<IndicesType>& indices_def,
int64_t axis = 0) {
return [input_def, indices_def, axis](ModelTestBuilder& builder,
std::vector<QuantParams<QuantType>>& output_qparams) {
NodeArg* input = MakeTestInput(builder, input_def);
QuantParams<QuantType> input_qparams = GetTestInputQuantParams<QuantType>(input_def);
NodeArg* input_qdq = AddQDQNodePair<QuantType>(builder, input, input_qparams.scale, input_qparams.zero_point);

NodeArg* indices = MakeTestInput(builder, indices_def);

NodeArg* gather_output = builder.MakeIntermediate();
Node& gather_node = builder.AddNode("Gather", {input_qdq, indices}, {gather_output});
gather_node.AddAttribute("axis", axis);

AddQDQNodePairWithOutputAsGraphOutput<QuantType>(builder, gather_output, output_qparams[0].scale, output_qparams[0].zero_point);
};
}

// Test the accuracy of a QDQ Gather model on QNN EP. Checks if the QDQ model on QNN EP as accurate as the QDQ model on CPU EP
// (compared to float32 model).
template <typename QuantType, typename IndicesType>
static void RunQDQGatherOpTest(const TestInputDef<float>& input_def, const TestInputDef<IndicesType>& indices_def,
int64_t axis, int opset, ExpectedEPNodeAssignment expected_ep_assignment) {
static void RunQDQGatherOpTest(const TestInputDef<float>& input_def,
const TestInputDef<IndicesType>& indices_def,
const std::vector<ONNX_NAMESPACE::AttributeProto>& attrs,
int opset,
ExpectedEPNodeAssignment expected_ep_assignment) {
ProviderOptions provider_options;
#if defined(_WIN32)
provider_options["backend_path"] = "QnnHtp.dll";
#else
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy<QuantType>(BuildGatherOpTestCase<IndicesType>(input_def, indices_def, axis),
BuildQDQGatherOpTestCase<QuantType, IndicesType>(input_def, indices_def, axis),
auto f32_model_builder = BuildOpTestCase<float, IndicesType>("Gather", {input_def}, {indices_def}, attrs);
auto qdq_model_builder = BuildQDQOpTestCase<QuantType, IndicesType>("Gather", {input_def}, {indices_def}, attrs);

TestQDQModelAccuracy<QuantType>(f32_model_builder,
qdq_model_builder,
provider_options,
opset,
expected_ep_assignment,
1e-5f);
expected_ep_assignment);
}

// Test creates a DQ -> Gather -> Q -> DQ graph, and checks that all
Expand All @@ -77,7 +47,7 @@ static void RunQDQGatherOpTest(const TestInputDef<float>& input_def, const TestI
TEST_F(QnnHTPBackendTests, GatherOp_IndicesStaticInt64_Axis0) {
RunQDQGatherOpTest<uint8_t, int64_t>(TestInputDef<float>({3, 2}, false, {1.0f, 1.2f, 2.3f, 3.4f, 4.5f, 5.7f}),
TestInputDef<int64_t>({2, 2}, true, {0, 1, 1, 2}),
0,
{utils::MakeAttribute("axis", static_cast<int64_t>(0))},
13,
ExpectedEPNodeAssignment::All);
}
Expand All @@ -86,7 +56,7 @@ TEST_F(QnnHTPBackendTests, GatherOp_IndicesStaticInt64_Axis0) {
TEST_F(QnnHTPBackendTests, GatherOp_IndicesDynamicInt64_Axis0) {
RunQDQGatherOpTest<uint8_t, int64_t>(TestInputDef<float>({3, 2}, false, {1.0f, 1.2f, 2.3f, 3.4f, 4.5f, 5.7f}),
TestInputDef<int64_t>({2, 2}, false, {0, 1, 1, 2}),
0,
{utils::MakeAttribute("axis", static_cast<int64_t>(0))},
13,
ExpectedEPNodeAssignment::None);
}
Expand All @@ -98,7 +68,7 @@ TEST_F(QnnHTPBackendTests, GatherOp_IndicesDynamicInt64_Axis0) {
TEST_F(QnnHTPBackendTests, GatherOp_IndicesStaticInt32_Axis0) {
RunQDQGatherOpTest<uint8_t, int32_t>(TestInputDef<float>({3, 2}, false, {1.0f, 1.2f, 2.3f, 3.4f, 4.5f, 5.7f}),
TestInputDef<int32_t>({2, 2}, true, {0, 1, 1, 2}),
0,
{utils::MakeAttribute("axis", static_cast<int64_t>(0))},
13,
ExpectedEPNodeAssignment::All);
}
Expand All @@ -110,7 +80,7 @@ TEST_F(QnnHTPBackendTests, GatherOp_IndicesStaticInt32_Axis0) {
TEST_F(QnnHTPBackendTests, GatherOp_IndicesDynamicInt32_Axis0) {
RunQDQGatherOpTest<uint8_t, int32_t>(TestInputDef<float>({3, 2}, false, {1.0f, 1.2f, 2.3f, 3.4f, 4.5f, 5.7f}),
TestInputDef<int32_t>({2, 2}, false, {0, 1, 1, 2}),
0,
{utils::MakeAttribute("axis", static_cast<int64_t>(0))},
13,
ExpectedEPNodeAssignment::All);
}
Expand All @@ -122,7 +92,7 @@ TEST_F(QnnHTPBackendTests, GatherOp_IndicesDynamicInt32_Axis0) {
TEST_F(QnnHTPBackendTests, GatherOp_IndicesStaticInt32_Axis1) {
RunQDQGatherOpTest<uint8_t, int32_t>(TestInputDef<float>({3, 3}, false, {1.0f, 1.2f, 1.9f, 2.3f, 3.4f, 3.9f, 4.5f, 5.7f, 5.9f}),
TestInputDef<int32_t>({1, 2}, true, {0, 2}),
1,
{utils::MakeAttribute("axis", static_cast<int64_t>(1))},
13,
ExpectedEPNodeAssignment::All);
}
Expand Down
4 changes: 2 additions & 2 deletions onnxruntime/test/providers/qnn/gemm_op_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ static void RunGemmTestOnCPU(const std::vector<TestInputDef<DataType>>& input_de
provider_options["backend_path"] = "libQnnCpu.so";
#endif

RunQnnModelTest(BuildOpTestCase("Gemm", input_defs, attrs),
RunQnnModelTest(BuildOpTestCase<float>("Gemm", input_defs, {}, attrs),
provider_options,
opset,
expected_ep_assignment);
Expand Down Expand Up @@ -192,7 +192,7 @@ static void RunQDQGemmTestOnHTP(const std::vector<TestInputDef<float>>& input_de
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy<InputAQType>(BuildOpTestCase<float>("Gemm", input_defs, attrs),
TestQDQModelAccuracy<InputAQType>(BuildOpTestCase<float>("Gemm", input_defs, {}, attrs),
BuildQDQGemmTestCase<InputAQType, InputBQType>(input_defs, attrs),
provider_options,
opset,
Expand Down
21 changes: 1 addition & 20 deletions onnxruntime/test/providers/qnn/instance_norm_htp_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,6 @@ namespace onnxruntime {
namespace test {
#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)

// Function that builds a float32 model with an InstanceNormalization operator.
GetTestModelFn BuildInstanceNormTestCase(const TestInputDef<float>& input_def,
const TestInputDef<float>& scale_def,
const TestInputDef<float>& bias_def,
const std::vector<ONNX_NAMESPACE::AttributeProto>& attrs) {
return [input_def, scale_def, bias_def, attrs](ModelTestBuilder& builder) {
NodeArg* input = MakeTestInput(builder, input_def);
NodeArg* scale = MakeTestInput(builder, scale_def);
NodeArg* bias = MakeTestInput(builder, bias_def);

NodeArg* output = builder.MakeOutput();
Node& op_node = builder.AddNode("InstanceNormalization", {input, scale, bias}, {output});

for (const auto& attr : attrs) {
op_node.AddAttributeProto(attr);
}
};
}

// Function that builds a QDQ model with an InstanceNormalization operator.
template <typename QuantType>
static GetTestQDQModelFn<QuantType> BuildQDQInstanceNormTestCase(const TestInputDef<float>& input_def,
Expand Down Expand Up @@ -93,7 +74,7 @@ static void RunInstanceNormQDQTest(const TestInputDef<float>& input_def,
#endif

// Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs.
TestQDQModelAccuracy(BuildInstanceNormTestCase(input_def, scale_def, bias_def, attrs),
TestQDQModelAccuracy(BuildOpTestCase<float>("InstanceNormalization", {input_def, scale_def, bias_def}, {}, attrs),
BuildQDQInstanceNormTestCase<QuantType>(input_def, scale_def, bias_def, attrs),
provider_options,
18,
Expand Down
4 changes: 2 additions & 2 deletions onnxruntime/test/providers/qnn/layer_norm_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ static void RunLayerNormCpuTest(const TestInputDef<float>& input_def,
provider_options["backend_path"] = "libQnnCpu.so";
#endif

RunQnnModelTest(BuildOpTestCase<float>("LayerNormalization", {input_def, scale_def}, attrs),
RunQnnModelTest(BuildOpTestCase<float>("LayerNormalization", {input_def, scale_def}, {}, attrs),
provider_options,
17,
expected_ep_assignment);
Expand Down Expand Up @@ -114,7 +114,7 @@ static void RunLayerNormQDQTest(const TestInputDef<float>& input_def,
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy(BuildOpTestCase<float>("LayerNormalization", {input_def, scale_def}, attrs),
TestQDQModelAccuracy(BuildOpTestCase<float>("LayerNormalization", {input_def, scale_def}, {}, attrs),
BuildQDQLayerNormTestCase<InputQType, ScaleQType>(input_def, scale_def, attrs),
provider_options,
17, // opset
Expand Down
46 changes: 7 additions & 39 deletions onnxruntime/test/providers/qnn/leakyrelu_op_htp_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#include <string>
#include "core/graph/graph.h"
#include "core/graph/node_attr_utils.h"

#include "test/optimizer/qdq_test_utils.h"
#include "test/providers/qnn/qnn_test_utils.h"
Expand All @@ -15,42 +16,10 @@ namespace onnxruntime {
namespace test {
#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)

// Creates a function that builds a model with a LeakyRelu operator.
static GetTestModelFn BuildLeakyReluOpTestCase(const TestInputDef<float>& input_def, float alpha) {
return [input_def, alpha](ModelTestBuilder& builder) {
NodeArg* input = MakeTestInput(builder, input_def);
NodeArg* output = builder.MakeOutput();
Node& leakyrelu_node = builder.AddNode("LeakyRelu", {input}, {output});
leakyrelu_node.AddAttribute("alpha", alpha);
};
}

// Creates a function that builds a QDQ model with a LeakyRelu operator.
template <typename QuantType>
static GetTestQDQModelFn<QuantType> BuildQDQLeakyReluOpTestCase(const TestInputDef<float>& input_def,
float alpha) {
return [input_def, alpha](ModelTestBuilder& builder,
std::vector<QuantParams<QuantType>>& output_qparams) {
// input => Q => DQ =>
NodeArg* input = MakeTestInput(builder, input_def);
QuantParams<QuantType> input_qparams = GetTestInputQuantParams<QuantType>(input_def);
NodeArg* input_qdq = AddQDQNodePair<QuantType>(builder, input, input_qparams.scale, input_qparams.zero_point);

// LeakryRelu
auto* leakyrelu_output = builder.MakeIntermediate();
Node& leakyrelu_node = builder.AddNode("LeakyRelu", {input_qdq}, {leakyrelu_output});
leakyrelu_node.AddAttribute("alpha", alpha);

// => Q => DQ -> final output
AddQDQNodePairWithOutputAsGraphOutput<QuantType>(builder, leakyrelu_output, output_qparams[0].scale,
output_qparams[0].zero_point);
};
}

// Checks the accuracy of a QDQ LeakyRelu model by comparing to ORT CPU EP.
template <typename QuantType>
static void RunLeakyReluOpQDQTest(const TestInputDef<float>& input_def,
float alpha,
const std::vector<ONNX_NAMESPACE::AttributeProto>& attrs,
int opset,
ExpectedEPNodeAssignment expected_ep_assignment) {
ProviderOptions provider_options;
Expand All @@ -60,12 +29,11 @@ static void RunLeakyReluOpQDQTest(const TestInputDef<float>& input_def,
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy(BuildLeakyReluOpTestCase(input_def, alpha),
BuildQDQLeakyReluOpTestCase<QuantType>(input_def, alpha),
TestQDQModelAccuracy(BuildOpTestCase<float>("LeakyRelu", {input_def}, {}, attrs),
BuildQDQOpTestCase<QuantType>("LeakyRelu", {input_def}, {}, attrs),
provider_options,
opset,
expected_ep_assignment,
1e-5f);
expected_ep_assignment);
}

// Test creates a DQ -> Gather -> Q -> DQ graph, and checks that all
Expand All @@ -74,7 +42,7 @@ static void RunLeakyReluOpQDQTest(const TestInputDef<float>& input_def,
// - Uses uint8 as the quantization type.
TEST_F(QnnHTPBackendTests, LeakyReluOpSet15) {
RunLeakyReluOpQDQTest<uint8_t>(TestInputDef<float>({1, 2, 3}, false, {-40.0f, -20.0f, 0.0f, 10.0f, 30.0f, 40.0f}),
0.2f,
{utils::MakeAttribute("alpha", 0.2f)},
15,
ExpectedEPNodeAssignment::All);
}
Expand All @@ -85,7 +53,7 @@ TEST_F(QnnHTPBackendTests, LeakyReluOpSet15) {
// - Uses uint8 as the quantization type.
TEST_F(QnnHTPBackendTests, LeakyReluOpSet16) {
RunLeakyReluOpQDQTest<uint8_t>(TestInputDef<float>({1, 2, 3}, false, {-40.0f, -20.0f, 0.0f, 10.0f, 30.0f, 40.0f}),
0.2f,
{utils::MakeAttribute("alpha", 0.2f)},
16,
ExpectedEPNodeAssignment::All);
}
Expand Down
9 changes: 4 additions & 5 deletions onnxruntime/test/providers/qnn/max_min_op_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ static void RunCPUMinOrMaxOpTest(const std::string& op_type,
provider_options["backend_path"] = "libQnnCpu.so";
#endif

RunQnnModelTest(BuildOpTestCase(op_type, input_defs, {}, kOnnxDomain),
RunQnnModelTest(BuildOpTestCase<float>(op_type, input_defs, {}, {}, kOnnxDomain),
provider_options,
opset,
expected_ep_assignment);
Expand All @@ -48,12 +48,11 @@ static void RunQDQMinOrMaxOpTest(const std::string& op_type,
provider_options["backend_path"] = "libQnnHtp.so";
#endif

TestQDQModelAccuracy(BuildOpTestCase(op_type, input_defs, {}, kOnnxDomain), // baseline float32 model
BuildQDQOpTestCase<QType>(op_type, input_defs, {}, kOnnxDomain), // QDQ model
TestQDQModelAccuracy(BuildOpTestCase<float>(op_type, input_defs, {}, {}, kOnnxDomain), // baseline float32 model
BuildQDQOpTestCase<QType>(op_type, input_defs, {}, {}, kOnnxDomain), // QDQ model
provider_options,
opset,
expected_ep_assignment,
1e-4f);
expected_ep_assignment);
}

//
Expand Down
Loading

0 comments on commit 80c8fb4

Please sign in to comment.