Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DynamicQuantizeMatMul test update #19517

Merged
merged 2 commits into from
Feb 16, 2024
Merged

DynamicQuantizeMatMul test update #19517

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c8dc2f7
DynamicQuantizeMatMul Test updates
raoanag Feb 14, 2024
d92e58d
UInt8_test_with_empty_input test updates
raoanag Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 141 additions & 62 deletions onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,90 @@
namespace test {

template <typename T>
void TestDynamicQuantizeMatMul(const std::vector<int64_t>& A_dims,
std::vector<int64_t> B_dims,
const std::string& reference_model,
bool is_matrix_b_constant,
static void CalculateDynamicQuantizeMatMul(const int64_t M, const int64_t N, const int64_t K,
const std::vector<float>& A_data, const std::vector<T>& B_data,
std::vector<float>& B_scale, std::vector<T>& B_zero_point,
const std::vector<float>& Bias, std::vector<float>& Y_data,
bool per_column, bool has_zp, bool has_bias) {
// DynamicQuantize Matrix A
const uint32_t num_elements = M * K;
std::vector<T> QuantA_data(num_elements);
std::vector<float> A_scale;
std::vector<T> A_zero_point;

// Get max and min
float min = std::numeric_limits<float>::max();
float max = std::numeric_limits<float>::lowest();
float qmax = static_cast<float>(std::numeric_limits<T>::max());
float qmin = static_cast<float>(std::numeric_limits<T>::lowest());

for (uint32_t i = 0; i < num_elements; ++i) {
max = std::max(A_data[i], max);
min = std::min(A_data[i], min);
}

// Adjust the maximum and minimum to include zero
max = std::max(max, 0.0f);
min = std::min(min, 0.0f);

float scale = static_cast<float>(max - min) / (qmax - qmin);
T zeroPoint = std::round(std::clamp(qmin - min / scale, qmin, qmax));

A_scale.push_back(scale);
A_zero_point.push_back(zeroPoint);

// Matrix Multiplication
for (uint32_t i = 0; i < num_elements; ++i) {
QuantA_data[i] = static_cast<T>(std::round((A_data[i] / scale) + zeroPoint));
}
if (!per_column) {
B_zero_point.resize(N, B_zero_point[0]);
B_scale.resize(N, B_scale[0]);
}

for (int64_t m = 0; m < M; m++) {
for (int64_t n = 0; n < N; n++) {
float sum = 0.0f;
for (int64_t k = 0; k < K; k++) {
float A_dequantized = (static_cast<int>(QuantA_data[m * K + k]) - static_cast<int>(A_zero_point[0]))
* A_scale[0];

float B_dequantized = has_zp ?

Check warning on line 74 in onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc

View workflow job for this annotation

GitHub Actions / Lint C++ 

[cpplint] reported by reviewdog 🐶
 Line ends in whitespace.  Consider deleting these extra spaces.  [whitespace/end_of_line] [4]

Raw Output:
onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc:74:  Line ends in whitespace.  Consider deleting these extra spaces.  [whitespace/end_of_line] [4]
(static_cast<int>(B_data[k * N + n]) - static_cast<int>(B_zero_point[n])) * B_scale[n] :
B_data[k * N + n] * B_scale[n];

sum += A_dequantized * B_dequantized;
}
if (has_bias) {
sum += Bias[n];
}
Y_data[m * N + n] = sum;
}
}
}

template <typename T>
void TestDynamicQuantizeMatMul(bool is_matrix_b_constant,
bool per_column = false,
bool has_zp = true,
bool has_bias = false) {
bool has_bias = false,
bool empty_input = false) {
// create rand inputs
RandomValueGenerator random{};

int64_t M = empty_input ? 1 : 4;
int64_t N = 128;
int64_t K = 128;
std::vector<int64_t> A_dims{empty_input ? 0 : M, K};
std::vector<int64_t> B_dims{K, N};
std::vector<int64_t> Y_dims{empty_input ? 0 : M, K};
std::vector<float> A_data = random.Uniform<float>(A_dims, -1.0f, 1.0f);

std::vector<T> B_data;
std::vector<int> tmp_B_data = random.Uniform<int32_t>(B_dims, std::numeric_limits<T>::min(), std::numeric_limits<T>::max());
std::vector<T> tmp_B_data = random.Uniform<T>(B_dims,
(constexpr(std::is_same_v<T, int8_t>)) ?
std::numeric_limits<int8_t>::lowest() / 2 :
std::numeric_limits<uint8_t>::lowest(),
std::numeric_limits<T>::max() / 2);
std::transform(tmp_B_data.begin(), tmp_B_data.end(), std::back_inserter(B_data), [](int32_t v) -> T {
return static_cast<T>(v);
});
Expand All @@ -47,7 +117,9 @@
std::for_each(B_zero_point.begin(),
B_zero_point.end(),
[&random](T& zp) {
zp = static_cast<T>(random.Uniform<int32_t>(std::array<int64_t, 1>{1}, std::numeric_limits<T>::min(), std::numeric_limits<T>::max())[0]);
zp = static_cast<T>(random.Uniform<T>(std::array<int64_t, 1>{1},
std::numeric_limits<T>::min(),
std::numeric_limits<T>::max())[0]);
});

std::vector<float> Bias = random.Uniform<float>(AsSpan({B_dims.back()}), -0.1f, 0.1f);
Expand All @@ -69,77 +141,84 @@
test.AddOptionalInputEdge<float>();
}

test.AddReferenceOutputs(reference_model);
std::vector<float> Y_data(M * N);
CalculateDynamicQuantizeMatMul<T>(M, N, K, A_data, B_data, B_scale, B_zero_point, Bias, Y_data,
per_column, has_zp, has_bias);
test.AddOutput<float>("Y", Y_dims, Y_data);
test.Run();
}

template <typename Scalar, bool HasZeroPoint, bool HasBias>
void RunDynamicQuantizeMatMulTest(const string& model_path) {
std::vector<int64_t> A_dims{4, 128};
std::vector<int64_t> B_dims{128, 128};
std::vector<int64_t> Y_dims{4, 128};

TestDynamicQuantizeMatMul<Scalar>(A_dims,
B_dims,
model_path,
false, /*is_matrix_b_constant*/
false, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
template <typename T, bool HasZeroPoint, bool HasBias>
void RunDynamicQuantizeMatMulTest() {
TestDynamicQuantizeMatMul<T>(false, /*is_matrix_b_constant*/
false, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
);

TestDynamicQuantizeMatMul<Scalar>(A_dims,
B_dims,
model_path,
true, /*is_matrix_b_constant*/
false, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
TestDynamicQuantizeMatMul<T>(true, /*is_matrix_b_constant*/
false, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
);

TestDynamicQuantizeMatMul<Scalar>(A_dims,
B_dims,
model_path,
false, /*is_matrix_b_constant*/
true, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
TestDynamicQuantizeMatMul<T>(false, /*is_matrix_b_constant*/
true, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
);

TestDynamicQuantizeMatMul<Scalar>(A_dims,
B_dims,
model_path,
true, /*is_matrix_b_constant*/
true, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
TestDynamicQuantizeMatMul<T>(true, /*is_matrix_b_constant*/
true, /*per_column*/
HasZeroPoint, /*has_zp*/
HasBias /*has_bias*/
);
}

TEST(DynamicQuantizeMatMul, HasZeroPoint_NoBias_test) {
RunDynamicQuantizeMatMulTest<int8_t, true, false>("testdata/dynamic_quantize_matmul_int8.onnx");
RunDynamicQuantizeMatMulTest<uint8_t, true, false>("testdata/dynamic_quantize_matmul_uint8.onnx");
TEST(DynamicQuantizeMatMul, HasZeroPoint_NoBias_test_S8) {
RunDynamicQuantizeMatMulTest<int8_t, true, false>();
}

TEST(DynamicQuantizeMatMul, NoZeroPoint_HasBias_test) {
RunDynamicQuantizeMatMulTest<int8_t, false, true>("testdata/dynamic_quantize_matmul_int8_bias.onnx");
RunDynamicQuantizeMatMulTest<uint8_t, false, true>("testdata/dynamic_quantize_matmul_uint8_bias.onnx");
TEST(DynamicQuantizeMatMul, HasZeroPoint_NoBias_test_U8) {
RunDynamicQuantizeMatMulTest<uint8_t, true, false>();
}

TEST(DynamicQuantizeMatMul, NoZeroPoint_HasBias_test_S8) {
RunDynamicQuantizeMatMulTest<int8_t, false, true>();
}

TEST(DynamicQuantizeMatMul, NoZeroPoint_HasBias_test_U8) {
RunDynamicQuantizeMatMulTest<uint8_t, false, true>();
}

TEST(DynamicQuantizeMatMul, NoZeroPoint_NoBias_test_S8) {
RunDynamicQuantizeMatMulTest<int8_t, false, false>();
}

TEST(DynamicQuantizeMatMul, NoZeroPoint_NoBias_test_U8) {
RunDynamicQuantizeMatMulTest<uint8_t, false, false>();
}

TEST(DynamicQuantizeMatMul, HasZeroPoint_HasBias_test_S8) {
RunDynamicQuantizeMatMulTest<int8_t, true, true>();
}

TEST(DynamicQuantizeMatMul, HasZeroPoint_HasBias_test_U8) {
RunDynamicQuantizeMatMulTest<uint8_t, true, true>();
}

TEST(DynamicQuantizeMatMul, UInt8_test_with_empty_input) {
std::vector<int64_t> A_dims{0, 128};
std::vector<int64_t> B_dims{128, 128};
std::vector<int64_t> Y_dims{0, 128};

TestDynamicQuantizeMatMul<uint8_t>(A_dims,
B_dims,
"testdata/dynamic_quantize_matmul_uint8.onnx",
false /*is_matrix_b_constant*/);

TestDynamicQuantizeMatMul<uint8_t>(A_dims,
B_dims,
"testdata/dynamic_quantize_matmul_uint8.onnx",
true /*is_matrix_b_constant*/);
std::vector<int64_t> A_dims{0, 2};
std::vector<int64_t> B_dims{2, 2};
std::vector<int64_t> Y_dims{0, 2};
OpTester test("DynamicQuantizeMatMul", 1, onnxruntime::kMSDomain);
test.AddInput<float>("T1", A_dims, {});
test.AddInput<uint8_t>("T2", B_dims, {1, 6, 0, 8});
test.AddInput<float>("b_scale", {1}, {1.0f});
test.AddInput<uint8_t>("b_zero_point", {1}, {0});
test.AddOptionalInputEdge<float>();
test.AddOutput<float>("Y", {0, 2}, {});
test.Run();
}

TEST(DynamicQuantizeMatMul, B_PerColumn_ND) {
Expand Down
Loading