From 491302c01766a0cccb491bb04a71bd0f3e04a657 Mon Sep 17 00:00:00 2001 From: SaeHie Park Date: Mon, 12 Aug 2024 22:49:38 +0000 Subject: [PATCH] [tflchef] Extract cook_operands method This will extract cook_operands method to gather tensors. ONE-DCO-1.0-Signed-off-by: SaeHie Park --- compiler/tflchef/core/src/ModelChef.cpp | 368 +++++++++++++++++++++++- 1 file changed, 364 insertions(+), 4 deletions(-) diff --git a/compiler/tflchef/core/src/ModelChef.cpp b/compiler/tflchef/core/src/ModelChef.cpp index b857e3ce5c9..03422a84471 100644 --- a/compiler/tflchef/core/src/ModelChef.cpp +++ b/compiler/tflchef/core/src/ModelChef.cpp @@ -186,6 +186,9 @@ class ModelChef void init(void); void cook(const ::tflchef::ModelRecipe &model_recipe); +private: + template void cook_operands(const T &graph); + template void cook_graph(const T &graph, std::map &symbol_table); @@ -206,6 +209,10 @@ class ModelChef // this is used to find tensor ID(index) with tensor name std::vector> _symbol_tables; + // per graph that needs clear afer graph is processed + // Operand-related + std::vector> _tensor_vec; + std::string _graph_name; }; @@ -250,15 +257,352 @@ make_dim_metadata_vec(flatbuffers::FlatBufferBuilder *flatbuffer_builder, int32_ return dim_metadata_vec; } +template void ModelChef::cook_operands(const T &graph) +{ + int32_t buffer_start = _buffer_vec.size(); + int32_t buffer_index = 0; + + // Create buffer(s) 1~n(I) for input(s) + const auto size_input = graph.input_size(); + for (int ci = 0; ci < size_input; ++ci) + { + tflite::BufferBuilder buffer_builder{*_flatbuffer_builder}; + _buffer_vec.emplace_back(buffer_builder.Finish()); + } + // Create buffer(s) n(I)+1~n(I)+n(O) for output(s) + const auto size_output = graph.output_size(); + for (int co = 0; co < size_output; ++co) + { + tflite::BufferBuilder buffer_builder{*_flatbuffer_builder}; + _buffer_vec.emplace_back(buffer_builder.Finish()); + } + + auto input_names = as_dataset(graph.input()).vectorize(); + auto output_names = as_dataset(graph.output()).vectorize(); + + for (const auto &operand : graph.operand()) + { + assert(operand.has_name()); + assert(operand.has_type()); + + flatbuffers::Offset sparsity_index; + + flatbuffers::Offset> shape; + std::vector dims; + if (operand.has_shape()) + { + dims = as_dims(operand.shape()); + shape = _flatbuffer_builder->CreateVector(dims); + } + + auto name = _flatbuffer_builder->CreateString(operand.name()); + + buffer_index = 0; + + // Create Buffer if filler is specified + if (operand.has_filler()) + { + const auto &filler = operand.filler(); + + assert(filler.has_tag()); + + auto args = ranged_arguments(filler.arg().begin(), filler.arg().end()); + auto chef = data_chef_registry(operand.type()).lookup(filler.tag()).create(args); + + assert(chef != nullptr); + + // Create Data + int32_t count = (element_count(dims) > 0) ? element_count(dims) : filler.arg_size(); + auto data_vec = chef->generate(count); + + if (operand.has_make_sparse() && operand.make_sparse()) + { + assert(not operand.has_sparsity()); + assert(operand.has_shape()); + assert(operand.type() != tflchef::TensorType::INT4); + + const int32_t dims_count = dims.size(); + std::vector traversal_order_vec; + std::vector format_vec; + for (int32_t o = 0; o < dims_count; ++o) + traversal_order_vec.push_back(o); + for (int32_t o = 0; o < dims_count - 1; ++o) + format_vec.push_back(sparsity::kTfLiteDimDense); + format_vec.push_back(sparsity::kTfLiteDimSparseCSR); + + if (operand.type() == tflchef::FLOAT32) + { + ::sparsity::FormatConverter converter(dims, traversal_order_vec, format_vec); + converter.DenseToSparse(reinterpret_cast(data_vec.data())); + const auto &sparse_data = converter.GetData(); + + std::vector sparse_uint8; + for (int c = 0; c < sparse_data.size(); ++c) + { + const float value = sparse_data.at(c); + const uint8_t *arr = reinterpret_cast(&value); + for (uint32_t b = 0; b < sizeof(float); ++b) + { + sparse_uint8.emplace_back(arr[b]); + } + } + auto data = _flatbuffer_builder->CreateVector(sparse_uint8); + + // Create Buffer + tflite::BufferBuilder buffer_builder{*_flatbuffer_builder}; + buffer_builder.add_data(data); + auto buffer = buffer_builder.Finish(); + + // Update Buffer Index & Vector + buffer_index = _buffer_vec.size(); + _buffer_vec.emplace_back(buffer); + + // save SparsityParameters + auto traversal_order = _flatbuffer_builder->CreateVector(traversal_order_vec); + + // Create block map + std::vector block_map_vec{}; + auto block_map = _flatbuffer_builder->CreateVector(block_map_vec); + + // Create dimension metadata + const auto &dim_metadata_src = converter.GetDimMetadata(); + auto dim_metadata_vec = + make_dim_metadata_vec(_flatbuffer_builder.get(), dims_count, traversal_order_vec, + format_vec, dim_metadata_src); + auto dim_metadata = _flatbuffer_builder->CreateVector(dim_metadata_vec); + sparsity_index = tflite::CreateSparsityParameters(*_flatbuffer_builder, traversal_order, + block_map, dim_metadata); + } + else if (operand.type() == tflchef::FLOAT16) + { + ::sparsity::FormatConverter converter(dims, traversal_order_vec, format_vec); + converter.DenseToSparse(reinterpret_cast(data_vec.data())); + const auto &sparse_data = converter.GetData(); + + std::vector sparse_uint8; + for (int c = 0; c < sparse_data.size(); ++c) + { + const uint16_t value = sparse_data.at(c); + const uint8_t *arr = reinterpret_cast(&value); + for (uint32_t b = 0; b < sizeof(uint16_t); ++b) + { + sparse_uint8.emplace_back(arr[b]); + } + } + auto data = _flatbuffer_builder->CreateVector(sparse_uint8); + + // Create Buffer + tflite::BufferBuilder buffer_builder{*_flatbuffer_builder}; + buffer_builder.add_data(data); + auto buffer = buffer_builder.Finish(); + + // Update Buffer Index & Vector + buffer_index = _buffer_vec.size(); + _buffer_vec.emplace_back(buffer); + + // save SparsityParameters + auto traversal_order = _flatbuffer_builder->CreateVector(traversal_order_vec); + + // Create block map + std::vector block_map_vec{}; + auto block_map = _flatbuffer_builder->CreateVector(block_map_vec); + + // Create dimension metadata + const auto &dim_metadata_src = converter.GetDimMetadata(); + auto dim_metadata_vec = + make_dim_metadata_vec(_flatbuffer_builder.get(), dims_count, traversal_order_vec, + format_vec, dim_metadata_src); + auto dim_metadata = _flatbuffer_builder->CreateVector(dim_metadata_vec); + sparsity_index = tflite::CreateSparsityParameters(*_flatbuffer_builder, traversal_order, + block_map, dim_metadata); + } + else + { + throw std::runtime_error{"NYI: unsupported operand type"}; + } + } + else + { + // pack for INT4 and replace data_vec + if (operand.type() == tflchef::TensorType::INT4) + { + uint32_t packed = (count + 1) / 2; + std::vector data_packed(packed); + for (uint32_t idx = 0; idx < packed; ++idx) + { + uint32_t sidx = idx * 2; + data_packed[idx] = data_vec[sidx++] & 0x0f; + if (sidx < count) + data_packed[idx] |= data_vec[sidx] << 4; + } + data_vec = data_packed; + } + + auto data = _flatbuffer_builder->CreateVector(data_vec); + + // Create Buffer + tflite::BufferBuilder buffer_builder{*_flatbuffer_builder}; + buffer_builder.add_data(data); + auto buffer = buffer_builder.Finish(); + + // Update Buffer Index & Vector + buffer_index = _buffer_vec.size(); + _buffer_vec.emplace_back(buffer); + } + } + else + { + // if this is input or output, assign to that buffer_index + int idx = 0; + for (auto it = input_names.begin(); it != input_names.end(); ++it, ++idx) + { + if (*it == operand.name()) + { + buffer_index = buffer_start + idx; + break; + } + } + if (buffer_index == 0) + { + idx = 0; + for (auto it = output_names.begin(); it != output_names.end(); ++it, ++idx) + { + if (*it == operand.name()) + { + buffer_index = buffer_start + size_input + idx; + break; + } + } + } + if (buffer_index == 0) + { + // we couldn't find the buffer; create an empty buffer for this tensor + buffer_index = _buffer_vec.size(); + + tflite::BufferBuilder buffer_builder{*_flatbuffer_builder}; + _buffer_vec.emplace_back(buffer_builder.Finish()); + } + } + assert(buffer_index != 0); + + flatbuffers::Offset quant_index; + + // Create QuantizationParameters if quant is specified + if (operand.has_quant()) + { + const auto &quant = operand.quant(); + + // Create each parameters + // NOTE if some parameters are not given, those will be set to default value + std::vector quant_max_vec(quant.max_size()); + std::vector quant_min_vec(quant.min_size()); + std::vector quant_scale_vec(quant.scale_size()); + std::vector quant_zero_point_vec(quant.zero_point_size()); + + for (uint32_t i = 0; i < quant.max_size(); ++i) + quant_max_vec.at(i) = quant.max(i); + for (uint32_t i = 0; i < quant.min_size(); ++i) + quant_min_vec.at(i) = quant.min(i); + for (uint32_t i = 0; i < quant.scale_size(); ++i) + quant_scale_vec.at(i) = quant.scale(i); + for (uint32_t i = 0; i < quant.zero_point_size(); ++i) + quant_zero_point_vec.at(i) = quant.zero_point(i); + + auto quant_max = _flatbuffer_builder->CreateVector(quant_max_vec); + auto quant_min = _flatbuffer_builder->CreateVector(quant_min_vec); + auto quant_scale = _flatbuffer_builder->CreateVector(quant_scale_vec); + auto quant_zero_point = _flatbuffer_builder->CreateVector(quant_zero_point_vec); + + // Create QuantizationParameters + tflite::QuantizationParametersBuilder quant_builder{*_flatbuffer_builder}; + quant_builder.add_max(quant_max); + quant_builder.add_min(quant_min); + quant_builder.add_scale(quant_scale); + quant_builder.add_zero_point(quant_zero_point); + quant_builder.add_quantized_dimension(quant.quantized_dimension()); + + // Update QuantizationParameters Index + quant_index = quant_builder.Finish(); + } + + if (operand.has_sparsity()) + { + const auto &sparsity = operand.sparsity(); + + // Create traversal order + std::vector traversal_order_vec{sparsity.traversal_order().dim().begin(), + sparsity.traversal_order().dim().end()}; + auto traversal_order = _flatbuffer_builder->CreateVector(traversal_order_vec); + + // Create block map + std::vector block_map_vec{sparsity.block_map().dim().begin(), + sparsity.block_map().dim().end()}; + auto block_map = _flatbuffer_builder->CreateVector(block_map_vec); + + // Create dimension metadata + std::vector> dim_metadata_vec; + auto recipe_dim_metadata = sparsity.dim_metadata(); + for (const auto &dm : recipe_dim_metadata) + { + // Create array segments + auto tflite_array_segments = + as_tflite_sparse_index_vec(*_flatbuffer_builder, dm.array_segments()); + + // Create array indices + auto tflite_array_indices = + as_tflite_sparse_index_vec(*_flatbuffer_builder, dm.array_indices()); + + auto tflite_dim_metadata_builder = tflite::DimensionMetadataBuilder{*_flatbuffer_builder}; + tflite_dim_metadata_builder.add_format(as_tflite_dimensiontype(dm.format())); + tflite_dim_metadata_builder.add_dense_size(dm.dense_size()); + tflite_dim_metadata_builder.add_array_segments(tflite_array_segments); + tflite_dim_metadata_builder.add_array_segments_type( + as_tflite_sparse_idx_vec_type(dm.array_segments().type())); + tflite_dim_metadata_builder.add_array_indices(tflite_array_indices); + tflite_dim_metadata_builder.add_array_indices_type( + as_tflite_sparse_idx_vec_type(dm.array_indices().type())); + auto tflite_dim_metadata = tflite_dim_metadata_builder.Finish(); + dim_metadata_vec.emplace_back(tflite_dim_metadata); + } + auto dim_metadata = _flatbuffer_builder->CreateVector(dim_metadata_vec); + + sparsity_index = tflite::CreateSparsityParameters(*_flatbuffer_builder, traversal_order, + block_map, dim_metadata); + } + + flatbuffers::Offset> shape_signature; + if (operand.has_shape_signature()) + { + auto signature = as_dims(operand.shape_signature()); + shape_signature = _flatbuffer_builder->CreateVector(signature); + } + + // Create Tensor + tflite::TensorBuilder tensor_builder{*_flatbuffer_builder}; + + tensor_builder.add_shape(shape); + tensor_builder.add_type(as_tflite_tensortype(operand.type())); + tensor_builder.add_buffer(buffer_index); + tensor_builder.add_name(name); + tensor_builder.add_is_variable(operand.is_variable()); + if (operand.has_quant()) + tensor_builder.add_quantization(quant_index); + tensor_builder.add_sparsity(sparsity_index); + if (operand.has_shape_signature()) + tensor_builder.add_shape_signature(shape_signature); + + // Append! + _tensor_vec.emplace_back(tensor_builder.Finish()); + } +} + template void ModelChef::cook_graph(const T &graph, std::map &symbol_table) { LOGGER(l); assert(symbol_table.empty()); // FIX_CALLER_UNLESS - - // Operand-related - std::vector> tensor_vec; + assert(_tensor_vec.empty()); // FIX_CALLER_UNLESS // Operation-related std::vector> operator_vec; @@ -280,6 +624,7 @@ void ModelChef::cook_graph(const T &graph, std::map &symbo } }; +#if 0 int32_t buffer_start = _buffer_vec.size(); int32_t buffer_index = 0; @@ -624,6 +969,20 @@ void ModelChef::cook_graph(const T &graph, std::map &symbo symbol_table[tensor_name] = tensor_index; } +#endif + + cook_operands(graph); + + for (const auto &operand : graph.operand()) + { + // Update Tensor Name -> Tensor Index Map + int32_t tensor_index = symbol_table.size(); + const auto &tensor_name = operand.name(); + + INFO(l) << "Symbol [" << tensor_name << "] = Tensor " << tensor_index << std::endl; + + symbol_table[tensor_name] = tensor_index; + } // Create Operator for (const auto &operation : graph.operation()) @@ -689,7 +1048,7 @@ void ModelChef::cook_graph(const T &graph, std::map &symbo std::vector output_vec = as_dataset(graph.output()).map(lookup).vectorize(); // Create "SubGraph" arguments - auto tensors = _flatbuffer_builder->CreateVector(tensor_vec); + auto tensors = _flatbuffer_builder->CreateVector(_tensor_vec); auto inputs = _flatbuffer_builder->CreateVector(input_vec); auto outputs = _flatbuffer_builder->CreateVector(output_vec); auto operators = _flatbuffer_builder->CreateVector(operator_vec); @@ -781,6 +1140,7 @@ void ModelChef::cook(const ::tflchef::ModelRecipe &model_recipe) _graph_name = stringStream.str(); symbol_table.clear(); + _tensor_vec.clear(); cook_graph<::tflchef::Graph>(graph, symbol_table); _symbol_tables.push_back(symbol_table); }