Merge master into 7.0 (#494)

* Updating linear resize dimensions check (#362)

* Updates global pooling functions to work correctly with dynamic shapes (#365)

* Support empty initializers for optional inputs (#366)

* Add support for empty initializers for optional inputs

* Alphabetize importPluginFactory

* Support ceiling mode padding for dynamic inputs (#368)

* Register empty Constant node outputs to support empty weights (#369)

* Update myelin library name on Windows (#371)

* Update logic to import ONNX initializers (#375)

* Adding more type checks (#380)

* Add type check for gather and shapedweights attribute imports (#384)

* Throw warning if seed input is provided for randomuniform nodes (#386)

* Update spacetodepth importer to support fulldims and dynamic shapes (#392)

* Add check to avoid console spam of warnings (#402)

* fix some build warnings/errors on Windows VS2019 (#403)

* remove c++11/14 non-compliant constexpr lambdas

* fix build warnings on VS2019

* disable shape input tensor

* Revert "disable shape input tensor"

This reverts commit 9a49e03.

* Support opset11 padding (#408)

* Fix loop importer scan output calculation (#412)

* Fix typo in operators.md supported onnx operators (#399)

There are two overlapping RNN operators, one supporting and the other not. Since onnx supports RNN, the one with supported N should be removed.

Signed-off-by: juhyung <[email protected]>

* Added optimization only mode which runs optimization passes on the model without converting it to tensorrt. (#420)

* New command line options.
* Updated documentation.
* Currently requires linking against onnx project.

* Support opset8 scan (#433)

* Fix deconv importer and remote instancenormalization epsilon clamp value (#434)

* Fix deconv importer and remote instancenormalization epsilon clamp value

* Remove dilations

* Add check for shape tensor outputs (#437)

* Fix slice caclulation for -INT_MAX (#438)

* Support boolean weight conversion to tensors (#439)

* Fix node output accesser for older versions of protobuf (#441)

* Add const qualifier to isNullTensor() (#446)

* Support negative slicing across an entire axis (#453)

* Keep track of Loop tensor mappings (#454)

* Fix fp16 weight import (#484)

* Fix GEMM import assertion (#485)

Co-authored-by: pranavm-nvidia <[email protected]>
Co-authored-by: George Wu <[email protected]>
Co-authored-by: JuHyung Son <[email protected]>
Co-authored-by: Dennis Sandler <[email protected]>

CMakeLists.txt

@@ -111,9 +111,15 @@ find_library(TENSORRT_LIBRARY_INFER nvinfer
 find_library(TENSORRT_LIBRARY_INFER_PLUGIN nvinfer_plugin
  HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
  PATH_SUFFIXES lib lib64 lib/x64)
-find_library(TENSORRT_LIBRARY_MYELIN myelin
- HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
- PATH_SUFFIXES lib lib64 lib/x64)
+if(WIN32)
+ find_library(TENSORRT_LIBRARY_MYELIN myelin64_1
+ HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
+ PATH_SUFFIXES lib lib64 lib/x64)
+else()
+ find_library(TENSORRT_LIBRARY_MYELIN myelin
+ HINTS ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
+ PATH_SUFFIXES lib lib64 lib/x64)
+endif()
 set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_MYELIN})
 MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}")
 find_package_handle_standard_args(
@@ -152,7 +158,7 @@ endif()
 # --------------------------------
 add_executable(onnx2trt ${EXECUTABLE_SOURCES})
 target_include_directories(onnx2trt PUBLIC ${ONNX_INCLUDE_DIRS})
-target_link_libraries(onnx2trt PUBLIC ${PROTOBUF_LIB} nvonnxparser_static ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) #${CUDA_LIBRARIES} 
+target_link_libraries(onnx2trt PUBLIC ${PROTOBUF_LIB} onnx nvonnxparser_static ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) #${CUDA_LIBRARIES} 
 
 # --------------------------------
 # API Tests

ImporterContext.hpp

@@ -48,6 +48,8 @@ class ImporterContext final : public IImporterContext
  mTensorNameCounts; // Keep track of how many times a tensor name shows up, to avoid duplicate naming in TRT.
  StringMap<size_t>
  mLayerNameCounts; // Keep track of how many times a tensor name shows up, to avoid duplicate naming in TRT.
+ std::unordered_set<std::string> mUnsupportedShapeTensors; // Container to hold any shape tensors that are the output of layers that do not support shape tensors.
+ StringMap<std::string> mLoopTensors; // Container to map subgraph tensors to their original outer graph names.
 public:
  ImporterContext(nvinfer1::INetworkDefinition* network, nvinfer1::ILogger* logger)
  : _network(network)
@@ -78,6 +80,14 @@ class ImporterContext final : public IImporterContext
  {
  return mLayerPrecisions;
  }
+ virtual std::unordered_set<std::string>& unsupportedShapeTensors() override
+ {
+ return mUnsupportedShapeTensors;
+ }
+ virtual StringMap<std::string>& loopTensors() override
+ {
+ return mLoopTensors;
+ }
 
  // This actually handles weights as well, but is named this way to be consistent with the tensors()
  virtual void registerTensor(TensorOrWeights tensor, const std::string& basename) override

ModelImporter.cpp

@@ -172,7 +172,8 @@ Status parseGraph(
  auto& output = outputs.at(i);
  ssOutputs << "[" << outputName << " -> " << output.shape() << "], ";
  // Note: This condition is to allow ONNX outputs to be ignored
- if (output && !outputName.empty())
+ // Always register output weights (even empty ones) as it may be mapped to an unused input
+ if ((output || output.is_weights()) && !outputName.empty())
  {
  ctx->registerTensor(std::move(output), outputName);
  }
@@ -331,11 +332,11 @@ bool ModelImporter::supportsModel(
  }
  }
  }
-
- auto checkForInput = [&input_node](::ONNX_NAMESPACE::NodeProto const& node) {
+ auto* ctx = &_importer_ctx;
+ auto checkForInput = [&input_node, &ctx](::ONNX_NAMESPACE::NodeProto const& node) {
  for (auto input : node.input())
  {
- if (input_node == input)
+ if (input_node == input || ctx->loopTensors()[input_node] == input)
  {
  return true;
  }
@@ -351,17 +352,23 @@ bool ModelImporter::supportsModel(
  cout << "Failed to sort model topologically, exiting ..." << endl;
  return false;
  }
+
  for (int node_idx : topological_order)
  {
  ::ONNX_NAMESPACE::NodeProto const& node = model.graph().node(node_idx);
+
  // Add the node to the subgraph if:
- // 1. Importer function is regestiered for the operator type
- // 2. It is not directly connected to an unsupported input
- // 3. Parsing did not hit an error on the node
+ // 1. Importer function is registered for the operator type
+ // 2. It is NOT directly connected to an unsupported input
+ // 3. Parsing did NOT hit an error on the node
+ // 4. Any shape tensor output is coming from a supported node
  bool registered = supportsOperator(node.op_type().c_str());
  bool containsInput = (input_node.empty()) ? false : checkForInput(node);
  bool containsIndex = node_idx == error_node;
- if (registered && !containsInput && !containsIndex)
+ auto const tensor = node.output(0);
+ bool supportedShapeTensorOutput = ctx->unsupportedShapeTensors().count(tensor) == 0 ? true : false;
+
+ if (registered && !containsInput && !containsIndex && supportedShapeTensorOutput)
  {
  if (newSubGraph)
  {
@@ -447,6 +454,18 @@ void removeShapeTensorCasts(IImporterContext* ctx)
  {
  t.setType(SHAPE_TENSOR_TYPE);
  }
+ // Some layers do not support shape tensor outputs. Keep track of these tensor names
+ // for supportsModel().
+ auto type = layer->getType();
+ auto elementwiseOp = layer->getType() == nvinfer1::LayerType::kELEMENTWISE ? (static_cast<nvinfer1::IElementWiseLayer*>(layer))->getOperation() : nvinfer1::ElementWiseOperation::kSUM;
+ auto reduceOp = layer->getType() == nvinfer1::LayerType::kREDUCE ? (static_cast<nvinfer1::IReduceLayer*>(layer))->getOperation() : nvinfer1::ReduceOperation::kSUM;
+
+ if (!supportsShapeTensor(type, elementwiseOp, reduceOp))
+ {
+ auto name = layer->getOutput(0)->getName();
+ ctx->unsupportedShapeTensors().insert(name);
+ LOG_ERROR("Found " << name << " as a shape tensor output from a layer that does not support it!");
+ }
  }
  }
 }

OnnxAttrs.cpp

@@ -109,7 +109,11 @@ onnx2trt::ShapedWeights OnnxAttrs::get<onnx2trt::ShapedWeights>(const std::strin
 {
  ::ONNX_NAMESPACE::TensorProto const& onnx_weights_tensor = this->at(key)->t();
  onnx2trt::ShapedWeights weights;
- convertOnnxWeights(onnx_weights_tensor, &weights, mCtx);
+ // Return empty weights if conversion failed
+ if (!convertOnnxWeights(onnx_weights_tensor, &weights, mCtx))
+ {
+ return onnx2trt::ShapedWeights::empty(::ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
+ }
  return weights;
 }
 

README.md

@@ -58,6 +58,15 @@ ONNX models can also be converted to human-readable text:
 
  onnx2trt my_model.onnx -t my_model.onnx.txt
 
+ONNX models can also be optimized by ONNX's optimization libraries.
+To optimize an ONNX model and output a new one use `-m` to specify the output model name and `-O` to specify a semicolon-separated list of optimization passes to apply:
+
+ onnx2trt my_model.onnx -O "pass_1;pass_2;pass_3" -m my_model_optimized.onnx
+
+See more all available optimization passes by running:
+
+ onnx2trt -p
+
 See more usage information by running:
 
  onnx2trt -h

TensorOrWeights.hpp

@@ -67,14 +67,18 @@ class TensorOrWeights
  {
  return _variant == NODE_WEIGHTS;
  }
+ bool isNullTensor() const
+ {
+ return is_tensor() && _tensor == nullptr;
+ }
  nvinfer1::ITensor& tensor()
  {
- assert(is_tensor());
+ assert(!isNullTensor());
  return *_tensor;
  }
  nvinfer1::ITensor const& tensor() const
  {
- assert(is_tensor());
+ assert(!isNullTensor());
  return *_tensor;
  }
  ShapedWeights& weights()
@@ -99,6 +103,10 @@ class TensorOrWeights
  {
  return is_tensor() ? _tensor->getType() == nvinfer1::DataType::kINT32 : _weights.type == ::ONNX_NAMESPACE::TensorProto_DataType_INT32;
  }
+ bool isBool() const
+ {
+ return is_tensor() ? _tensor->getType() == nvinfer1::DataType::kBOOL : _weights.type == ::ONNX_NAMESPACE::TensorProto_DataType_BOOL;
+ }
 };
 
 } // namespace onnx2trt