[Wait for #2794, #2795][Graph] add inplace setting through layer property

nntrainer/layers/acti_func.h

@@ -37,8 +37,8 @@ class ActiFunc {
    */
   template <typename T = float>
   ActiFunc(ActivationType at = ActivationType::ACT_NONE,
-           bool in_place_ = true) :
-    in_place(in_place_) {
+           bool is_inplace_ = true) :
+    is_inplace(is_inplace_) {
     setActiFunc<T>(at);
   }
 
@@ -72,19 +72,19 @@ class ActiFunc {
       this->setActivation<T>(leakyRelu<T>, leakyReluPrime<T>);
       break;
     case ActivationType::ACT_SWISH:
-      in_place = false;
+      is_inplace = false;
       this->setActivation<Tensor>(swish<T>, swishPrime<T>);
       break;
     case ActivationType::ACT_GELU:
-      in_place = false;
+      is_inplace = false;
       this->setActivation<Tensor>(gelu<T>, geluPrime<T>);
       break;
     case ActivationType::ACT_TANH_GELU:
-      in_place = false;
+      is_inplace = false;
       this->setActivation<Tensor>(tanhGelu<T>, tanhGeluPrime<T>);
       break;
     case ActivationType::ACT_SIGMOID_GELU:
-      in_place = false;
+      is_inplace = false;
       this->setActivation<Tensor>(sigmoidGelu<T>, sigmoidGeluPrime<T>);
       break;
     case ActivationType::ACT_ELU:
@@ -149,7 +149,7 @@ class ActiFunc {
   /**
    * @copydoc Layer::supportInPlace()
    */
-  bool supportInPlace() const { return in_place; }
+  bool supportInPlace() const { return is_inplace; }
 
   /**
    * @brief       Calculate softmax for Tensor Type
@@ -649,7 +649,7 @@ class ActiFunc {
       &activation_fn,
     std::function<funcParam &(funcParam const &, funcParam const &, funcParam &,
                               funcParam const &)> const &activation_prime_fn) {
-    if (in_place)
+    if (is_inplace)
       return ML_ERROR_INVALID_PARAMETER;
 
     _act_fn = activation_fn;
@@ -672,7 +672,7 @@ class ActiFunc {
       &activation_fn,
     std::function<funcParam &(funcParam &, funcParam &)> const
       &activation_prime_fn) {
-    if (!in_place) {
+    if (!is_inplace) {
       _act_prime_fn = [activation_prime_fn](
                         funcParam const &t_in, funcParam &t_out,
                         funcParam &outgoing_derivative,
@@ -715,7 +715,7 @@ class ActiFunc {
     _act_fn = [activation_fn](Tensor const &x, Tensor &hidden) -> Tensor & {
       return x.apply(activation_fn, hidden);
     };
-    if (!in_place) {
+    if (!is_inplace) {
       _act_prime_fn =
         [activation_prime_fn](Tensor const &t_in, Tensor &t_out,
                               Tensor &outgoing_derivative,
@@ -765,7 +765,7 @@ class ActiFunc {
       throw std::runtime_error(
         "Error setting activation layer to work in-place");
 
-    in_place = val;
+    is_inplace = val;
   }
 
 private:
@@ -780,7 +780,7 @@ class ActiFunc {
 
   ActivationType
     activation_type; /**< type of the activation represented by this */
-  bool in_place;     /**< if this class should operate in_place */
+  bool is_inplace;   /**< if this class should operate is_inplace */
 };
 
 } // namespace nntrainer

nntrainer/layers/add_layer.cpp

@@ -11,6 +11,7 @@
  *
  */
 
+#include "common_properties.h"
 #include <add_layer.h>
 #include <nntrainer_error.h>
 #include <nntrainer_log.h>

nntrainer/layers/add_layer.h

@@ -35,7 +35,8 @@ class AddLayer : public BinaryOperationLayer {
   /**
    * @brief Constructor of Add Layer
    */
-  AddLayer() : BinaryOperationLayer(), add_props(props::Print()) {}
+  AddLayer() :
+    BinaryOperationLayer(), add_props(props::Print(), props::InPlaceProp()) {}
 
   /**
    *  @brief  Move constructor of Add Layer.
@@ -74,6 +75,23 @@ class AddLayer : public BinaryOperationLayer {
    */
   bool supportBackwarding() const final { return true; };
 
+  /**
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
+   */
+  InPlaceType initializeInPlace() final {
+    if (std::get<props::InPlaceProp>(add_props).empty() ||
+        std::get<props::InPlaceProp>(add_props).get()) {
+      is_inplace = true;
+    } else {
+      is_inplace = false;
+    }
+    if (!supportInPlace())
+      return InPlaceType::NONE;
+    else
+      return InPlaceType::NON_RESTRICTING;
+  }
+
   /**
    * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods
    * method)
@@ -91,7 +109,7 @@ class AddLayer : public BinaryOperationLayer {
    */
   const std::string getType() const final { return AddLayer::type; }
 
-  std::tuple<props::Print> add_props;
+  std::tuple<props::Print, props::InPlaceProp> add_props;
 
   inline static const std::string type = "add";
 };

nntrainer/layers/bn_layer.h

@@ -100,6 +100,15 @@ class BatchNormalizationLayer : public Layer {
    */
   bool supportBackwarding() const override { return true; }
 
+  /**
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
+   */
+  InPlaceType initializeInPlace() final {
+    is_inplace = true;
+    return InPlaceType::NON_RESTRICTING;
+  }
+
   using Layer::setProperty;
 
   /**
@@ -108,11 +117,6 @@ class BatchNormalizationLayer : public Layer {
    */
   void setProperty(const std::vector<std::string> &values) override;
 
-  /**
-   * @copydoc Layer::supportInPlace()
-   */
-  bool supportInPlace() const override { return true; }
-
   /**
    * @copydoc Layer::setBatch(RunLayerContext &context, unsigned int batch)
    */

nntrainer/layers/cl_layers/reshape_cl.h

@@ -87,9 +87,13 @@ class ReshapeLayerCl : public Layer {
   bool supportBackwarding() const override { return false; };
 
   /**
-   * @copydoc Layer::supportInPlace()
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
    */
-  bool supportInPlace() const override { return true; }
+  InPlaceType initializeInPlace() final {
+    is_inplace = true;
+    return InPlaceType::NON_RESTRICTING;
+  }
 
   /**
    * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods

nntrainer/layers/common_properties.h

@@ -124,6 +124,16 @@ class TensorDimension : public TensorDimProperty {
   using prop_tag = dimension_prop_tag;      /**< property type */
 };
 
+/**
+ * @brief Inplace operation property
+ *
+ */
+class InPlaceProp : public nntrainer::Property<bool> {
+public:
+  static constexpr const char *key = "inplace"; /**< unique key to access */
+  using prop_tag = bool_prop_tag;               /**< property type */
+};
+
 /**
  * @brief trainable property, use this to set and check how if certain layer is
  * trainable

nntrainer/layers/divide_layer.h

@@ -30,7 +30,10 @@ class DivideLayer : public BinaryOperationLayer {
   /**
    * @brief Constructor of Divide Layer
    */
-  DivideLayer() : BinaryOperationLayer(), divide_props(props::Print()) {}
+  DivideLayer() :
+    BinaryOperationLayer(),
+    divide_props(props::Print(), props::InPlaceProp()),
+    support_backwarding(true) {}
 
   /**
    * @brief Destructor of Divide Layer
@@ -72,7 +75,27 @@ class DivideLayer : public BinaryOperationLayer {
   /**
    * @copydoc bool supportBackwarding() const
    */
-  bool supportBackwarding() const final { return true; };
+  bool supportBackwarding() const final { return support_backwarding; };
+
+  /**
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
+   */
+  InPlaceType initializeInPlace() final {
+    if (std::get<props::InPlaceProp>(divide_props).empty() ||
+        !std::get<props::InPlaceProp>(divide_props).get()) {
+      is_inplace = false;
+      support_backwarding = true;
+    } else {
+      is_inplace = true;
+      support_backwarding = false;
+    }
+
+    if (!supportInPlace())
+      return InPlaceType::NONE;
+    else
+      return InPlaceType::NON_RESTRICTING;
+  }
 
   /**
    * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods
@@ -91,7 +114,8 @@ class DivideLayer : public BinaryOperationLayer {
    */
   const std::string getType() const final { return DivideLayer::type; };
 
-  std::tuple<props::Print> divide_props;
+  std::tuple<props::Print, props::InPlaceProp> divide_props;
+  bool support_backwarding; /**< support backwarding */
 
   inline static const std::string type = "divide";
 };

nntrainer/layers/dropout.h

@@ -30,9 +30,7 @@ class DropOutLayer : public Layer {
    * @brief     Constructor of DropOut Layer
    */
   DropOutLayer(float dropout = 0.0) :
-    Layer(),
-    dropout_rate(props::DropOutRate(dropout)),
-    epsilon(1e-3) {}
+    Layer(), dropout_rate(props::DropOutRate(dropout)), epsilon(1e-3) {}
 
   /**
    * @brief     Destructor of DropOut Layer
@@ -89,13 +87,6 @@ class DropOutLayer : public Layer {
    */
   void setProperty(const std::vector<std::string> &values) override;
 
-  /**
-   * @copydoc Layer::supportInPlace()
-   *
-   * @todo Enable in-place support once supported by manager
-   */
-  bool supportInPlace() const override { return false; }
-
   inline static const std::string type = "dropout";
 
 private:

nntrainer/layers/flatten_layer.h

@@ -28,8 +28,9 @@ class FlattenLayer : public ReshapeLayer {
   /**
    * @brief     Constructor of Flatten Layer
    */
-  FlattenLayer() : ReshapeLayer(), flatten_props(
-    props::StartDimension(), props::EndDimension()) {}
+  FlattenLayer() :
+    ReshapeLayer(),
+    flatten_props(props::StartDimension(), props::EndDimension()) {}
 
   /**
    * @brief     Destructor of Flatten Layer
@@ -58,6 +59,15 @@ class FlattenLayer : public ReshapeLayer {
    */
   void setProperty(const std::vector<std::string> &values) override;
 
+  /**
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
+   */
+  InPlaceType initializeInPlace() final {
+    is_inplace = true;
+    return InPlaceType::RESTRICTING;
+  }
+
   /**
    * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods
    * method)

nntrainer/layers/identity_layer.h

@@ -21,6 +21,7 @@ namespace nntrainer {
 
 /**
  * @class   Identity Layer
+ * @brief   Identity Layer
  * @note    Identity layers takes multiple tensors as input, redirects to output
  * without doing nothing (or if unavoidable, copying)
  */
@@ -69,9 +70,13 @@ class IdentityLayer final : public Layer {
   bool supportBackwarding() const override { return true; };
 
   /**
-   * @copydoc Layer::supportInPlace()
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
    */
-  bool supportInPlace() const override { return true; }
+  InPlaceType initializeInPlace() final {
+    is_inplace = true;
+    return InPlaceType::RESTRICTING;
+  }
 
   /**
    * @copydoc Layer::getType()

nntrainer/layers/input_layer.cpp

@@ -33,9 +33,7 @@ namespace nntrainer {
 static constexpr size_t SINGLE_INOUT_IDX = 0;
 
 InputLayer::InputLayer() :
-  Layer(),
-  input_props(props::Normalization(), props::Standardization()),
-  is_inplace(true) {}
+  Layer(), input_props(props::Normalization(), props::Standardization()) {}
 
 void InputLayer::setProperty(const std::vector<std::string> &values) {
   auto remain_props = loadProperties(values, input_props);
@@ -75,17 +73,6 @@ void InputLayer::finalize(InitLayerContext &context) {
   }
 
   context.setOutputDimensions(output_dims);
-
-  is_inplace = true;
-
-  /**
-   * @note Input Layer assuems that the FP32 IN Tensor always. Therefore, if the
-   * activation data type is not fp32, then it does not support in-place
-   * operation.
-   */
-  if (context.getActivationDataType() != ml::train::TensorDim::DataType::FP32) {
-    is_inplace = false;
-  }
 }
 
 } /* namespace nntrainer */

nntrainer/layers/input_layer.h

@@ -80,9 +80,13 @@ class InputLayer : public Layer {
   bool supportBackwarding() const override { return false; };
 
   /**
-   * @copydoc Layer::supportInPlace()
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
    */
-  bool supportInPlace() const override { return is_inplace; }
+  InPlaceType initializeInPlace() final {
+    is_inplace = true;
+    return InPlaceType::NON_RESTRICTING;
+  }
 
   /**
    * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods
@@ -105,7 +109,6 @@ class InputLayer : public Layer {
 
 private:
   std::tuple<props::Normalization, props::Standardization> input_props;
-  bool is_inplace;
 };
 } // namespace nntrainer