Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update xformer with conv redesign changes #838

Merged
merged 2 commits into from
Sep 21, 2023
Merged

Update xformer with conv redesign changes #838

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7b01cd8
Update xformer with conv redesign changes
panickal-xmos Sep 20, 2023
ca3afd8
Update submodules
panickal-xmos Sep 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion third_party/lib_nn
Open in desktop
Submodule lib_nn updated 28 files
+64 −45 lib_nn/api/AbstractKernel.hpp
+49 −300 lib_nn/api/AggregateFn.hpp
+0 −95 lib_nn/api/Conv2d.hpp
+0 −166 lib_nn/api/Filter2D.hpp
+46 −135 lib_nn/api/MemCpyFn.hpp
+45 −128 lib_nn/api/OutputTransformFn.hpp
+0 −39 lib_nn/api/Serialisable.hpp
+1 −1 lib_nn/src/asm/aggregate_fn_direct_binary.S
+1 −1 lib_nn/src/asm/aggregate_fn_direct_int8.S
+2 −2 lib_nn/src/asm/aggregate_fn_direct_int8_dw.S
+1 −1 lib_nn/src/asm/aggregate_fn_generic_binary.S
+1 −1 lib_nn/src/asm/aggregate_fn_generic_int8.S
+34 −41 lib_nn/src/cpp/AggregateFn.cpp
+18 −18 lib_nn/src/cpp/AggregateFn_DW.cpp
+43 −59 lib_nn/src/cpp/Filter2D.cpp
+76 −83 lib_nn/src/cpp/MemCpyFn.cpp
+22 −22 lib_nn/src/cpp/OutputTransformFn.cpp
+0 −1 test/gtests/src/op/ref/test_Conv2dDenseReference.cpp
+0 −97 test/gtests/src/test_AbstractKernel.cpp
+0 −188 test/gtests/src/test_Filter2D.cpp
+0 −71 test/gtests/src/test_Serialisation.cpp
+13 −13 test/unit_test/src/main.c
+31 −44 test/unit_test/src/test_AggregateFns.cpp
+110 −108 test/unit_test/src/test_Conv2dRegression.cpp
+69 −64 test/unit_test/src/test_Conv2dRegressionBinary.cpp
+72 −65 test/unit_test/src/test_Conv2dRegression_DW.cpp
+9 −11 test/unit_test/src/test_MemCpyFns.cpp
+12 −20 test/unit_test/src/test_OutputTransforms.cpp
2 changes: 1 addition & 1 deletion third_party/lib_tflite_micro
Open in desktop
Submodule lib_tflite_micro updated 3 files
+150 −167 lib_tflite_micro/src/tflite-xcore-kernels/xcore_conv2d_v2.cc
+12 −16 lib_tflite_micro/src/tflite-xcore-kernels/xcore_strided_slice.cc
+1 −1 repos.list
13 changes: 8 additions & 5 deletions xformer/Transforms/ConvPatterns.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
#include "IR/XCoreOps.h"

#include "larq_compute_engine/mlir/ir/lce_ops.h"
#include "lib_nn/api/Conv2d.hpp"
#include "lib_nn/api/AbstractKernel.hpp"
#include "lib_nn/api/AggregateFn.hpp"
#include "lib_nn/api/MemCpyFn.hpp"
#include "lib_nn/api/OutputTransformFn.hpp"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "tensorflow/compiler/mlir/lite/ir/tfl_ops.h"
Expand Down Expand Up @@ -244,16 +247,16 @@ class ReplaceDepthwiseConv2DPattern
std::vector<int16_t> &mulsBiasesData) const;
};

template <typename Filter2DParams>
llvm::SmallVector<std::string> getAbstractKernelParamsForMultipleThreads(
static llvm::SmallVector<std::string> getAbstractKernelParamsForMultipleThreads(
llvm::SmallVector<std::array<int, 4>> imageRegionSplits,
const nn::ImageGeometry &Y) {
llvm::SmallVector<std::string> abstractKernelParams;
for (auto &regionsplits : imageRegionSplits) {
auto ir = nn::ImageRegion(regionsplits[0], regionsplits[1], 0,
regionsplits[2], regionsplits[3], Y.depth);
Filter2DParams akParams(Y, ir, VPU_INT8_ACC_PERIOD);
std::string akpStr = akParams.template serialise<Filter2DParams>();
nn::AbstractKernel ak(Y, ir, VPU_INT8_ACC_PERIOD);
auto akParams = ak.getParams();
std::string akpStr = std::string((char *)&akParams, sizeof(akParams));
abstractKernelParams.push_back(akpStr);
}
return abstractKernelParams;
Expand Down
74 changes: 42 additions & 32 deletions xformer/Transforms/ConvPatternsLCE.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,14 +281,16 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidDirectBinaryParams(
llvm::SmallVector<std::string> &abstractKernelParams,
std::vector<int8_t> &weightsData, std::vector<int16_t> &thresholdsData,
int &scratchBytes) const {
nn::DerefInputFn::Params imToColParams(args.X, args.K);
nn::DerefInputFn imToCol(args.X, args.K);
auto imToColParams = imToCol.getParams();

std::array<int, 4> filterShape = {args.outputDepth, args.filterHeight,
args.filterWidth, args.inputDepth};
nn::Conv2dReorderedWeights rw = nn::MatMulInt8::reorder_kernel_weights(
(int8_t *)args.filter.data(), filterShape, 1, args.padValue);

nn::MatMulBinaryDirectFn::Params afParams(args.X, args.K, args.inputDepth);
nn::MatMulBinaryDirectFn af(args.X, args.K, args.inputDepth);
auto afParams = af.getParams();

// adjust the thresholds from xorpopcount space
// to xcore space
Expand All @@ -298,13 +300,13 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidDirectBinaryParams(
nn::OutputTransformFn::pad_final_access(adjustedThresholds, VPU_INT16_EPV,
(int16_t)args.padValue);

std::string mfStr = imToColParams.serialise<nn::DerefInputFn::Params>();
std::string afStr = afParams.serialise<nn::MatMulBinaryDirectFn::Params>();
std::string otStr = ""; // otParams.serialise<nn::OT_int8::Params>();
std::string mfStr =
std::string((char *)&imToColParams, sizeof(imToColParams));
std::string afStr = std::string((char *)&afParams, sizeof(afParams));
std::string otStr = "";

abstractKernelParams =
getAbstractKernelParamsForMultipleThreads<nn::Filter2D::Params>(
args.imageRegionSplits, args.Y);
getAbstractKernelParamsForMultipleThreads(args.imageRegionSplits, args.Y);
strParams.push_back(mfStr);
strParams.push_back(afStr);
strParams.push_back(otStr);
Expand All @@ -320,7 +322,8 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidIndirectBinaryParams(
llvm::SmallVector<std::string> &abstractKernelParams,
std::vector<int8_t> &weightsData, std::vector<int16_t> &thresholdsData,
int &scratchBytes) const {
nn::ImToColValid::Params imToColParams(args.X, args.K, args.inputDepth);
nn::ImToColValid imToCol(args.X, args.K, args.inputDepth);
auto imToColParams = imToCol.getParams();

std::array<int, 4> filterShape = {args.outputDepth, args.filterHeight,
args.filterWidth, args.inputDepth};
Expand All @@ -330,7 +333,8 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidIndirectBinaryParams(
const int elementsPerByte = 8;
int inputBytes =
args.filterHeight * args.filterWidth * args.inputDepth / elementsPerByte;
nn::MatMulBinary::Params afParams(args.outputDepth, inputBytes);
nn::MatMulBinary af(args.outputDepth, inputBytes);
auto afParams = af.getParams();

// adjust the thresholds from xorpopcount space
// to xcore space
Expand All @@ -340,13 +344,13 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidIndirectBinaryParams(
nn::OutputTransformFn::pad_final_access(adjustedThresholds, VPU_INT16_EPV,
(int16_t)args.padValue);

std::string mfStr = imToColParams.serialise<nn::ImToColValid::Params>();
std::string afStr = afParams.serialise<nn::MatMulBinary::Params>();
std::string otStr = ""; // otParams.serialise<nn::OT_int8::Params>();
std::string mfStr =
std::string((char *)&imToColParams, sizeof(imToColParams));
std::string afStr = std::string((char *)&afParams, sizeof(afParams));
std::string otStr = "";

abstractKernelParams =
getAbstractKernelParamsForMultipleThreads<nn::Filter2D::Params>(
args.imageRegionSplits, args.Y);
getAbstractKernelParamsForMultipleThreads(args.imageRegionSplits, args.Y);
strParams.push_back(mfStr);
strParams.push_back(afStr);
strParams.push_back(otStr);
Expand All @@ -362,14 +366,16 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidDirectInt8Params(
llvm::SmallVector<std::string> &abstractKernelParams,
std::vector<int8_t> &weightsData, std::vector<int16_t> &mulsBiasesData,
int &scratchBytes) const {
nn::DerefInputFn::Params imToColParams(args.X, args.K);
nn::DerefInputFn imToCol(args.X, args.K);
auto imToColParams = imToCol.getParams();

std::array<int, 4> filterShape = {args.outputDepth, args.filterHeight,
args.filterWidth, args.inputDepth};
nn::Conv2dReorderedWeights rw = nn::MatMulInt8::reorder_kernel_weights(
(int8_t *)args.filter.data(), filterShape, 1, args.padValue);

nn::MatMulBinaryDirectFn::Params afParams(args.X, args.K, args.inputDepth);
nn::MatMulBinaryDirectFn af(args.X, args.K, args.inputDepth);
auto afParams = af.getParams();

int receptiveVolume = args.filterHeight * args.filterWidth * args.inputDepth;
nn::MulsAndBias mulAndBiases = nn::OT_int8_clamped::canonicalise_mul_and_bias(
Expand All @@ -388,16 +394,17 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidDirectInt8Params(
VPU_INT16_EPV,
(int16_t)args.padValue);

nn::OT_int8_clamped::Params otParams((int32_t)args.outputDepth,
qp.initial_shr, qp.final_shr);
nn::OT_int8_clamped ot((int32_t)args.outputDepth, qp.initial_shr,
qp.final_shr);
auto otParams = ot.getParams();

std::string mfStr = imToColParams.serialise<nn::DerefInputFn::Params>();
std::string afStr = afParams.serialise<nn::MatMulBinaryDirectFn::Params>();
std::string otStr = otParams.serialise<nn::OT_int8_clamped::Params>();
std::string mfStr =
std::string((char *)&imToColParams, sizeof(imToColParams));
std::string afStr = std::string((char *)&afParams, sizeof(afParams));
std::string otStr = std::string((char *)&otParams, sizeof(otParams));

abstractKernelParams =
getAbstractKernelParamsForMultipleThreads<nn::Filter2D::Params>(
args.imageRegionSplits, args.Y);
getAbstractKernelParamsForMultipleThreads(args.imageRegionSplits, args.Y);
strParams.push_back(mfStr);
strParams.push_back(afStr);
strParams.push_back(otStr);
Expand All @@ -413,7 +420,8 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidIndirectInt8Params(
llvm::SmallVector<std::string> &abstractKernelParams,
std::vector<int8_t> &weightsData, std::vector<int16_t> &mulsBiasesData,
int &scratchBytes) const {
nn::ImToColValid::Params imToColParams(args.X, args.K, args.inputDepth);
nn::ImToColValid imToCol(args.X, args.K, args.inputDepth);
auto imToColParams = imToCol.getParams();

std::array<int, 4> filterShape = {args.outputDepth, args.filterHeight,
args.filterWidth, args.inputDepth};
Expand All @@ -424,7 +432,8 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidIndirectInt8Params(
int inputBytes =
args.filterHeight * args.filterWidth * args.inputDepth / elementsPerByte;

nn::MatMulBinary::Params afParams(args.outputDepth, inputBytes);
nn::MatMulBinary af(args.outputDepth, inputBytes);
auto afParams = af.getParams();

int receptiveVolume = args.filterHeight * args.filterWidth * args.inputDepth;
nn::MulsAndBias mulAndBiases = nn::OT_int8_clamped::canonicalise_mul_and_bias(
Expand All @@ -443,16 +452,17 @@ LogicalResult ReplaceBConv2DPattern::getBConv2DValidIndirectInt8Params(
VPU_INT16_EPV,
(int16_t)args.padValue);

nn::OT_int8_clamped::Params otParams((int32_t)args.outputDepth,
qp.initial_shr, qp.final_shr);
nn::OT_int8_clamped ot((int32_t)args.outputDepth, qp.initial_shr,
qp.final_shr);
auto otParams = ot.getParams();

std::string mfStr = imToColParams.serialise<nn::ImToColValid::Params>();
std::string afStr = afParams.serialise<nn::MatMulBinary::Params>();
std::string otStr = otParams.serialise<nn::OT_int8_clamped::Params>();
std::string mfStr =
std::string((char *)&imToColParams, sizeof(imToColParams));
std::string afStr = std::string((char *)&afParams, sizeof(afParams));
std::string otStr = std::string((char *)&otParams, sizeof(otParams));

abstractKernelParams =
getAbstractKernelParamsForMultipleThreads<nn::Filter2D::Params>(
args.imageRegionSplits, args.Y);
getAbstractKernelParamsForMultipleThreads(args.imageRegionSplits, args.Y);
strParams.push_back(mfStr);
strParams.push_back(afStr);
strParams.push_back(otStr);
Expand Down
Loading