From a22f6b150e372df1cc55297b48f82b81b7d0c442 Mon Sep 17 00:00:00 2001 From: Brad Hards Date: Sun, 1 Sep 2024 20:51:33 +1000 Subject: [PATCH 01/13] plugins: refactor NALU parsing for shared future usage --- libheif/plugins/CMakeLists.txt | 4 +- libheif/plugins/decoder_ffmpeg.cc | 93 ++++++++----------------------- libheif/plugins/nalu_utils.cc | 82 +++++++++++++++++++++++++++ libheif/plugins/nalu_utils.h | 61 ++++++++++++++++++++ 4 files changed, 168 insertions(+), 72 deletions(-) create mode 100644 libheif/plugins/nalu_utils.cc create mode 100644 libheif/plugins/nalu_utils.h diff --git a/libheif/plugins/CMakeLists.txt b/libheif/plugins/CMakeLists.txt index 67f3e74ead..01564808dd 100644 --- a/libheif/plugins/CMakeLists.txt +++ b/libheif/plugins/CMakeLists.txt @@ -110,7 +110,9 @@ plugin_compilation(vvenc vvenc vvenc_FOUND VVENC VVENC) target_sources(heif PRIVATE encoder_mask.h - encoder_mask.cc) + encoder_mask.cc + nalu_utils.h + nalu_utils.cc) if (WITH_UNCOMPRESSED_CODEC) target_sources(heif PRIVATE diff --git a/libheif/plugins/decoder_ffmpeg.cc b/libheif/plugins/decoder_ffmpeg.cc index 98b4e101ee..63926b3615 100644 --- a/libheif/plugins/decoder_ffmpeg.cc +++ b/libheif/plugins/decoder_ffmpeg.cc @@ -21,12 +21,13 @@ #include "libheif/heif.h" #include "libheif/heif_plugin.h" #include "decoder_ffmpeg.h" +#include "nalu_utils.h" #if defined(HAVE_CONFIG_H) #include "config.h" #endif -#include + #include #include @@ -35,34 +36,10 @@ extern "C" #include } -class NalUnit { -public: - NalUnit(); - ~NalUnit(); - bool set_data(const unsigned char* in_data, int n); - int size() const { return nal_data_size; } - int unit_type() const { return nal_unit_type; } - const unsigned char* data() const { return nal_data_ptr; } - int bitExtracted(int number, int bits_count, int position_nr) - { - return (((1 << bits_count) - 1) & (number >> (position_nr - 1))); - } -private: - const unsigned char* nal_data_ptr; - int nal_unit_type; - int nal_data_size; -}; struct ffmpeg_decoder { - #define NAL_UNIT_VPS_NUT 32 - #define NAL_UNIT_SPS_NUT 33 - #define NAL_UNIT_PPS_NUT 34 - #define NAL_UNIT_IDR_W_RADL 19 - #define NAL_UNIT_IDR_N_LP 20 - - std::map> NalMap; - + NalMap nalMap; bool strict_decoding = false; }; @@ -148,43 +125,17 @@ bool NalUnit::set_data(const unsigned char* in_data, int n) return true; } -static struct heif_error ffmpeg_v1_push_data(void* decoder_raw, const void* data, size_t size) +static struct heif_error ffmpeg_v1_push_data(void *decoder_raw, const void *data, size_t size) { struct ffmpeg_decoder* decoder = (struct ffmpeg_decoder*) decoder_raw; const uint8_t* cdata = (const uint8_t*) data; - size_t ptr = 0; - while (ptr < size) { - if (4 > size - ptr) { - struct heif_error err = { heif_error_Decoder_plugin_error, - heif_suberror_End_of_data, - "insufficient data" }; - return err; - } - - uint32_t nal_size = (cdata[ptr] << 24) | (cdata[ptr + 1] << 16) | (cdata[ptr + 2] << 8) | (cdata[ptr + 3]); - ptr += 4; - - if (nal_size > size - ptr) { - struct heif_error err = { heif_error_Decoder_plugin_error, - heif_suberror_End_of_data, - "insufficient data" }; - return err; - } - - std::unique_ptr nal_unit = std::unique_ptr(new NalUnit()); - nal_unit->set_data(cdata + ptr, nal_size); - - // overwrite NalMap (frees old NalUnit, if it was set) - decoder->NalMap[nal_unit->unit_type()] = std::move(nal_unit); + return decoder->nalMap.parseHevcNalu(cdata, size); +} - ptr += nal_size; - } - return heif_error_success; -} static heif_chroma ffmpeg_get_chroma_format(enum AVPixelFormat pix_fmt) { if (pix_fmt == AV_PIX_FMT_GRAY8) @@ -342,19 +293,19 @@ static struct heif_error ffmpeg_v1_decode_image(void* decoder_raw, const unsigned char* heif_pps_data; const unsigned char* heif_idrpic_data; - if ((decoder->NalMap.count(NAL_UNIT_VPS_NUT) > 0) - && (decoder->NalMap.count(NAL_UNIT_SPS_NUT) > 0) - && (decoder->NalMap.count(NAL_UNIT_PPS_NUT) > 0) + if ((decoder->nalMap.count(NAL_UNIT_VPS_NUT) > 0) + && (decoder->nalMap.count(NAL_UNIT_SPS_NUT) > 0) + && (decoder->nalMap.count(NAL_UNIT_PPS_NUT) > 0) ) { - heif_vps_size = decoder->NalMap[NAL_UNIT_VPS_NUT]->size(); - heif_vps_data = decoder->NalMap[NAL_UNIT_VPS_NUT]->data(); + heif_vps_size = decoder->nalMap.size(NAL_UNIT_VPS_NUT); + heif_vps_data = decoder->nalMap.data(NAL_UNIT_VPS_NUT); - heif_sps_size = decoder->NalMap[NAL_UNIT_SPS_NUT]->size(); - heif_sps_data = decoder->NalMap[NAL_UNIT_SPS_NUT]->data(); + heif_sps_size = decoder->nalMap.size(NAL_UNIT_SPS_NUT); + heif_sps_data = decoder->nalMap.data(NAL_UNIT_SPS_NUT); - heif_pps_size = decoder->NalMap[NAL_UNIT_PPS_NUT]->size(); - heif_pps_data = decoder->NalMap[NAL_UNIT_PPS_NUT]->data(); + heif_pps_size = decoder->nalMap.size(NAL_UNIT_PPS_NUT); + heif_pps_data = decoder->nalMap.data(NAL_UNIT_PPS_NUT); } else { @@ -364,17 +315,17 @@ static struct heif_error ffmpeg_v1_decode_image(void* decoder_raw, return err; } - if ((decoder->NalMap.count(NAL_UNIT_IDR_W_RADL) > 0) || (decoder->NalMap.count(NAL_UNIT_IDR_N_LP) > 0)) + if ((decoder->nalMap.count(NAL_UNIT_IDR_W_RADL) > 0) || (decoder->nalMap.count(NAL_UNIT_IDR_N_LP) > 0)) { - if (decoder->NalMap.count(NAL_UNIT_IDR_W_RADL) > 0) + if (decoder->nalMap.count(NAL_UNIT_IDR_W_RADL) > 0) { - heif_idrpic_data = decoder->NalMap[NAL_UNIT_IDR_W_RADL]->data(); - heif_idrpic_size = decoder->NalMap[NAL_UNIT_IDR_W_RADL]->size(); + heif_idrpic_data = decoder->nalMap.data(NAL_UNIT_IDR_W_RADL); + heif_idrpic_size = decoder->nalMap.size(NAL_UNIT_IDR_W_RADL); } else { - heif_idrpic_data = decoder->NalMap[NAL_UNIT_IDR_N_LP]->data(); - heif_idrpic_size = decoder->NalMap[NAL_UNIT_IDR_N_LP]->size(); + heif_idrpic_data = decoder->nalMap.data(NAL_UNIT_IDR_N_LP); + heif_idrpic_size = decoder->nalMap.size(NAL_UNIT_IDR_N_LP); } } else @@ -416,7 +367,7 @@ static struct heif_error ffmpeg_v1_decode_image(void* decoder_raw, memcpy(hevc_data_ptr, heif_idrpic_data, heif_idrpic_size); // decoder->NalMap not needed anymore - decoder->NalMap.clear(); + decoder->nalMap.clear(); const AVCodec* hevc_codec = NULL; AVCodecParserContext* hevc_parser = NULL; diff --git a/libheif/plugins/nalu_utils.cc b/libheif/plugins/nalu_utils.cc new file mode 100644 index 0000000000..b1a5295168 --- /dev/null +++ b/libheif/plugins/nalu_utils.cc @@ -0,0 +1,82 @@ +/* + * HEIF codec. + * Copyright (c) 2023 Dirk Farin + * + * This file is part of libheif. + * + * libheif is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * libheif is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with libheif. If not, see . + */ + +#include +#include +#include "nalu_utils.h" + + +int NalUnit::bitExtracted(int number, int bits_count, int position_nr) +{ + return (((1 << bits_count) - 1) & (number >> (position_nr - 1))); +} + +size_t NalMap::count(int nal_type) +{ + return map.count(nal_type); +} + +const unsigned char* NalMap::data(int nal_type) +{ + return map[nal_type]->data(); +} + +int NalMap::size(int nal_type) +{ + return map[nal_type]->size(); +} + +const heif_error NalMap::parseHevcNalu(const uint8_t *cdata, size_t size) +{ + size_t ptr = 0; + while (ptr < size) + { + if (4 > size - ptr) + { + struct heif_error err = {heif_error_Decoder_plugin_error, + heif_suberror_End_of_data, + "insufficient data"}; + return err; + } + + uint32_t nal_size = (cdata[ptr] << 24) | (cdata[ptr + 1] << 16) | (cdata[ptr + 2] << 8) | (cdata[ptr + 3]); + ptr += 4; + + if (nal_size > size - ptr) + { + struct heif_error err = {heif_error_Decoder_plugin_error, + heif_suberror_End_of_data, + "insufficient data"}; + return err; + } + + std::unique_ptr nal_unit = std::unique_ptr(new NalUnit()); + nal_unit->set_data(cdata + ptr, nal_size); + + // overwrite NalMap (frees old NalUnit, if it was set) + map[nal_unit->unit_type()] = std::move(nal_unit); + + ptr += nal_size; + } + + return heif_error_success; +} + +void NalMap::clear() { map.clear(); } \ No newline at end of file diff --git a/libheif/plugins/nalu_utils.h b/libheif/plugins/nalu_utils.h new file mode 100644 index 0000000000..2ed75d073c --- /dev/null +++ b/libheif/plugins/nalu_utils.h @@ -0,0 +1,61 @@ +/* + * HEIF codec. + * Copyright (c) 2023 Dirk Farin + * + * This file is part of libheif. + * + * libheif is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * libheif is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with libheif. If not, see . + */ + +#include +#include +#include "libheif/heif.h" + +static const int NAL_UNIT_VPS_NUT = 32; +static const int NAL_UNIT_SPS_NUT = 33; +static const int NAL_UNIT_PPS_NUT = 34; +static const int NAL_UNIT_IDR_W_RADL = 19; +static const int NAL_UNIT_IDR_N_LP = 20; + +class NalUnit +{ +public: + NalUnit(); + ~NalUnit(); + bool set_data(const unsigned char* in_data, int n); + int size() const { return nal_data_size; } + int unit_type() const { return nal_unit_type; } + const unsigned char* data() const { return nal_data_ptr; } + int bitExtracted(int number, int bits_count, int position_nr); +private: + const unsigned char* nal_data_ptr; + int nal_unit_type; + int nal_data_size; +}; + +class NalMap +{ +public: + size_t count(int nal_type); + + const unsigned char* data(int nal_type); + + int size(int nal_type); + + const heif_error parseHevcNalu(const uint8_t *cdata, size_t size); + + void clear(); +private: + std::map> map; +}; \ No newline at end of file From fa06d8257774e6d6ee97a8a222d1f0218572568b Mon Sep 17 00:00:00 2001 From: Brad Hards Date: Sun, 1 Sep 2024 20:56:40 +1000 Subject: [PATCH 02/13] nalu: add forgotten include --- libheif/plugins/nalu_utils.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libheif/plugins/nalu_utils.cc b/libheif/plugins/nalu_utils.cc index b1a5295168..ca805ec374 100644 --- a/libheif/plugins/nalu_utils.cc +++ b/libheif/plugins/nalu_utils.cc @@ -20,6 +20,7 @@ #include #include +#include #include "nalu_utils.h" From 52fc1861b557bea9ea83281db5777d7387375286 Mon Sep 17 00:00:00 2001 From: Brad Hards Date: Mon, 2 Sep 2024 20:42:02 +1000 Subject: [PATCH 03/13] nalu: move remaining content --- libheif/plugins/decoder_ffmpeg.cc | 20 -------------------- libheif/plugins/nalu_utils.cc | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/libheif/plugins/decoder_ffmpeg.cc b/libheif/plugins/decoder_ffmpeg.cc index 63926b3615..c18502d086 100644 --- a/libheif/plugins/decoder_ffmpeg.cc +++ b/libheif/plugins/decoder_ffmpeg.cc @@ -105,26 +105,6 @@ void ffmpeg_set_strict_decoding(void* decoder_raw, int flag) decoder->strict_decoding = flag; } -NalUnit::NalUnit() -{ - nal_data_ptr = NULL; - nal_unit_type = 0; - nal_data_size = 0; -} - -NalUnit::~NalUnit() -{ - -} - -bool NalUnit::set_data(const unsigned char* in_data, int n) -{ - nal_data_ptr = in_data; - nal_unit_type = bitExtracted(nal_data_ptr[0], 6, 2); - nal_data_size = n; - return true; -} - static struct heif_error ffmpeg_v1_push_data(void *decoder_raw, const void *data, size_t size) { diff --git a/libheif/plugins/nalu_utils.cc b/libheif/plugins/nalu_utils.cc index ca805ec374..7a8495c81e 100644 --- a/libheif/plugins/nalu_utils.cc +++ b/libheif/plugins/nalu_utils.cc @@ -23,6 +23,25 @@ #include #include "nalu_utils.h" +NalUnit::NalUnit() +{ + nal_data_ptr = NULL; + nal_unit_type = 0; + nal_data_size = 0; +} + +NalUnit::~NalUnit() +{ + +} + +bool NalUnit::set_data(const unsigned char* in_data, int n) +{ + nal_data_ptr = in_data; + nal_unit_type = bitExtracted(nal_data_ptr[0], 6, 2); + nal_data_size = n; + return true; +} int NalUnit::bitExtracted(int number, int bits_count, int position_nr) { From 851e939aefd340bccfdbf1e9f513b9f2cd62e434 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 14:51:25 +0200 Subject: [PATCH 04/13] add ImageItem_AVC --- libheif/api/libheif/heif.h | 3 +- libheif/codecs/avc.cc | 143 +++++++++++++++++++++++++++++++++++ libheif/codecs/avc.h | 33 ++++++++ libheif/codecs/image_item.cc | 4 + libheif/context.cc | 2 +- 5 files changed, 183 insertions(+), 2 deletions(-) diff --git a/libheif/api/libheif/heif.h b/libheif/api/libheif/heif.h index aa075c87bd..5f91490740 100644 --- a/libheif/api/libheif/heif.h +++ b/libheif/api/libheif/heif.h @@ -218,7 +218,8 @@ enum heif_suberror_code heif_suberror_Invalid_pixi_box = 130, - heif_suberror_No_av1C_box = 131, + heif_suberror_No_av1C_box = 131, // deprecated: typo in name. Use 'heif_suberror_No_avcC_box' instead. + heif_suberror_No_avcC_box = 131, heif_suberror_Wrong_tile_image_pixel_depth = 132, diff --git a/libheif/codecs/avc.cc b/libheif/codecs/avc.cc index 730613df10..a6cbfdb165 100644 --- a/libheif/codecs/avc.cc +++ b/libheif/codecs/avc.cc @@ -23,6 +23,9 @@ #include #include #include +#include "file.h" +#include "context.h" + Error Box_avcC::parse(BitstreamRange &range) { m_configuration.configuration_version = range.read8(); @@ -142,3 +145,143 @@ std::string Box_avcC::profileIndicationAsText() const { return "Unknown"; } } + + +void Box_avcC::get_headers(std::vector& data) const +{ + for (const auto& sps : m_sps) { + data.insert(data.end(), sps.begin(), sps.end()); + } + + for (const auto& pps : m_pps) { + data.insert(data.end(), pps.begin(), pps.end()); + } +} + + + +Result ImageItem_AVC::encode(const std::shared_ptr& image, + struct heif_encoder* encoder, + const struct heif_encoding_options& options, + enum heif_image_input_class input_class) +{ +#if 0 + CodedImageData codedImage; + + auto hvcC = std::make_shared(); + + heif_image c_api_image; + c_api_image.image = image; + + struct heif_error err = encoder->plugin->encode_image(encoder->encoder, &c_api_image, input_class); + if (err.code) { + return Error(err.code, + err.subcode, + err.message); + } + + int encoded_width = 0; + int encoded_height = 0; + + for (;;) { + uint8_t* data; + int size; + + encoder->plugin->get_compressed_data(encoder->encoder, &data, &size, nullptr); + + if (data == nullptr) { + break; + } + + + const uint8_t NAL_SPS = 33; + + if ((data[0] >> 1) == NAL_SPS) { + Box_hvcC::configuration config; + + parse_sps_for_hvcC_configuration(data, size, &config, &encoded_width, &encoded_height); + + hvcC->set_configuration(config); + + codedImage.encoded_image_width = encoded_width; + codedImage.encoded_image_height = encoded_height; + } + + switch (data[0] >> 1) { + case 0x20: + case 0x21: + case 0x22: + hvcC->append_nal_data(data, size); + break; + + default: + codedImage.append_with_4bytes_size(data, size); + // m_heif_file->append_iloc_data_with_4byte_size(image_id, data, size); + } + } + + if (!encoded_width || !encoded_height) { + return Error(heif_error_Encoder_plugin_error, + heif_suberror_Invalid_image_size); + } + + codedImage.properties.push_back(hvcC); + + + // Make sure that the encoder plugin works correctly and the encoded image has the correct size. + + if (encoder->plugin->plugin_api_version >= 3 && + encoder->plugin->query_encoded_size != nullptr) { + uint32_t check_encoded_width = image->get_width(), check_encoded_height = image->get_height(); + + encoder->plugin->query_encoded_size(encoder->encoder, + image->get_width(), image->get_height(), + &check_encoded_width, + &check_encoded_height); + + assert((int)check_encoded_width == encoded_width); + assert((int)check_encoded_height == encoded_height); + } + + return codedImage; +#endif + assert(false); // TODO + return {}; +} + +Result> ImageItem_AVC::read_bitstream_configuration_data(heif_item_id itemId) const +{ + std::vector data; + + auto avcC_box = get_file()->get_property(itemId); + if (!avcC_box) { + return Error{heif_error_Invalid_input, + heif_suberror_No_av1C_box}; + } + + avcC_box->get_headers(data); + + return data; +} + + +int ImageItem_AVC::get_luma_bits_per_pixel() const +{ + auto avcC_box = get_file()->get_property(get_id()); + if (avcC_box) { + return 8; // TODO avcC_box->get_configuration().bit_depth_luma; + } + + return -1; +} + + +int ImageItem_AVC::get_chroma_bits_per_pixel() const +{ + auto avcC_box = get_file()->get_property(get_id()); + if (avcC_box) { + return 8; // TODO avcC_box->get_configuration().bit_depth_chroma; + } + + return -1; +} diff --git a/libheif/codecs/avc.h b/libheif/codecs/avc.h index 953bf54b01..fe27be17a7 100644 --- a/libheif/codecs/avc.h +++ b/libheif/codecs/avc.h @@ -26,6 +26,7 @@ #include #include #include +#include "codecs/image_item.h" class Box_avcC : public Box { public: @@ -61,6 +62,7 @@ class Box_avcC : public Box { return m_pps; } + void get_headers(std::vector& data) const; std::string dump(Indent &) const override; @@ -77,4 +79,35 @@ class Box_avcC : public Box { std::vector< std::vector > m_pps; }; + +class ImageItem_AVC : public ImageItem +{ +public: + ImageItem_AVC(HeifContext* ctx, heif_item_id id) : ImageItem(ctx, id) {} + + ImageItem_AVC(HeifContext* ctx) : ImageItem(ctx) {} + + const char* get_infe_type() const override { return "avc1"; } + + const char* get_auxC_alpha_channel_type() const override { return "urn:mpeg:mpegB:cicp:systems:auxiliary:alpha"; } + + const heif_color_profile_nclx* get_forced_output_nclx() const override { return nullptr; } + + heif_compression_format get_compression_format() const override { return heif_compression_AVC; } + + int get_luma_bits_per_pixel() const override; + + int get_chroma_bits_per_pixel() const override; + +protected: + Result> read_bitstream_configuration_data(heif_item_id itemId) const override; + +public: + + Result encode(const std::shared_ptr& image, + struct heif_encoder* encoder, + const struct heif_encoding_options& options, + enum heif_image_input_class input_class) override; +}; + #endif diff --git a/libheif/codecs/image_item.cc b/libheif/codecs/image_item.cc index 56ba3c004e..fa969a0175 100644 --- a/libheif/codecs/image_item.cc +++ b/libheif/codecs/image_item.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,9 @@ std::shared_ptr ImageItem::alloc_for_infe_box(HeifContext* ctx, const else if (item_type == "vvc1") { return std::make_shared(ctx, id); } + else if (item_type == "avc1") { + return std::make_shared(ctx, id); + } #if WITH_UNCOMPRESSED_CODEC else if (item_type == "unci") { return std::make_shared(ctx, id); diff --git a/libheif/context.cc b/libheif/context.cc index 04ad9b2d5f..b102f47041 100644 --- a/libheif/context.cc +++ b/libheif/context.cc @@ -237,7 +237,7 @@ static bool item_type_is_image(const std::string& item_type, const std::string& item_type == "tild" || item_type == "iden" || item_type == "iovl" || - item_type == "av01" || + item_type == "avc1" || item_type == "unci" || item_type == "vvc1" || item_type == "jpeg" || From e5fa8bc4340233690afd62a2427fa2412df8f2a4 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 15:11:37 +0200 Subject: [PATCH 05/13] restore av01 image type --- libheif/context.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libheif/context.cc b/libheif/context.cc index b102f47041..8a8b96f9db 100644 --- a/libheif/context.cc +++ b/libheif/context.cc @@ -233,6 +233,7 @@ std::string HeifContext::debug_dump_boxes() const static bool item_type_is_image(const std::string& item_type, const std::string& content_type) { return (item_type == "hvc1" || + item_type == "av01" || item_type == "grid" || item_type == "tild" || item_type == "iden" || From 5b587daf3c973917cb3d3eb9a27b2cbed4182bf1 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 15:19:52 +0200 Subject: [PATCH 06/13] fix: av1C != avcC --- libheif/api/libheif/heif.h | 5 +++-- libheif/error.cc | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libheif/api/libheif/heif.h b/libheif/api/libheif/heif.h index 5f91490740..ecd9c7eb23 100644 --- a/libheif/api/libheif/heif.h +++ b/libheif/api/libheif/heif.h @@ -218,8 +218,7 @@ enum heif_suberror_code heif_suberror_Invalid_pixi_box = 130, - heif_suberror_No_av1C_box = 131, // deprecated: typo in name. Use 'heif_suberror_No_avcC_box' instead. - heif_suberror_No_avcC_box = 131, + heif_suberror_No_av1C_box = 131, heif_suberror_Wrong_tile_image_pixel_depth = 132, @@ -247,6 +246,8 @@ enum heif_suberror_code // icbr is only needed in some situations, this error is for those cases heif_suberror_No_icbr_box = 142, + heif_suberror_No_avcC_box = 143, + // Decompressing generic compression or header compression data failed (e.g. bitstream corruption) heif_suberror_Decompression_invalid_data = 150, diff --git a/libheif/error.cc b/libheif/error.cc index caa2bed943..52ee52bf62 100644 --- a/libheif/error.cc +++ b/libheif/error.cc @@ -106,6 +106,8 @@ const char* Error::get_error_string(heif_suberror_code err) return "No 'vvcC' box"; case heif_suberror_No_av1C_box: return "No 'av1C' box"; + case heif_suberror_No_avcC_box: + return "No 'avcC' box"; case heif_suberror_No_pitm_box: return "No 'pitm' box"; case heif_suberror_No_ipco_box: From 25b499f4bf98b824b2f2d7b7294aeef2e249528a Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 15:21:07 +0200 Subject: [PATCH 07/13] add missing include --- libheif/codecs/avc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libheif/codecs/avc.h b/libheif/codecs/avc.h index fe27be17a7..bfb8a1bc86 100644 --- a/libheif/codecs/avc.h +++ b/libheif/codecs/avc.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "codecs/image_item.h" class Box_avcC : public Box { From 85b77353993da50cd3b51f6dbbd3ec1df90a7cd6 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 15:35:05 +0200 Subject: [PATCH 08/13] add heif_suberror_No_avcC_box to emscripten --- libheif/api/libheif/heif_emscripten.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libheif/api/libheif/heif_emscripten.h b/libheif/api/libheif/heif_emscripten.h index 42e65d563b..8c91316671 100644 --- a/libheif/api/libheif/heif_emscripten.h +++ b/libheif/api/libheif/heif_emscripten.h @@ -359,6 +359,7 @@ EMSCRIPTEN_BINDINGS(libheif) { .value("heif_suberror_Invalid_grid_data", heif_suberror_Invalid_grid_data) .value("heif_suberror_Missing_grid_images", heif_suberror_Missing_grid_images) .value("heif_suberror_No_av1C_box", heif_suberror_No_av1C_box) + .value("heif_suberror_No_avcC_box", heif_suberror_No_avcC_box) .value("heif_suberror_Invalid_clean_aperture", heif_suberror_Invalid_clean_aperture) .value("heif_suberror_Invalid_overlay_data", heif_suberror_Invalid_overlay_data) .value("heif_suberror_Overlay_image_outside_of_canvas", heif_suberror_Overlay_image_outside_of_canvas) From 94ba065e8c131e4207f3bf8816342364526cf22b Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 15:48:01 +0200 Subject: [PATCH 09/13] add heif_suberror_No_avcC_box to Go --- go/heif/heif.go | 2 + libheif/plugins/decoder_openh264.cc | 324 ++++++++++++++++++++++++++++ libheif/plugins/decoder_openh264.h | 34 +++ 3 files changed, 360 insertions(+) create mode 100644 libheif/plugins/decoder_openh264.cc create mode 100644 libheif/plugins/decoder_openh264.h diff --git a/go/heif/heif.go b/go/heif/heif.go index 0010e0fb2d..e2ec4de51c 100644 --- a/go/heif/heif.go +++ b/go/heif/heif.go @@ -248,6 +248,8 @@ const ( SuberrorNoAV1CBox = C.heif_suberror_No_av1C_box + SuberrorNoAVCCBox = C.heif_suberror_No_avcC_box + SuberrorInvalidCleanAperture = C.heif_suberror_Invalid_clean_aperture // Invalid specification of overlay image diff --git a/libheif/plugins/decoder_openh264.cc b/libheif/plugins/decoder_openh264.cc new file mode 100644 index 0000000000..901bf92214 --- /dev/null +++ b/libheif/plugins/decoder_openh264.cc @@ -0,0 +1,324 @@ +/* + * JPEG codec. + * Copyright (c) 2023 Dirk Farin + * + * This file is part of libheif. + * + * libheif is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * libheif is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with libheif. If not, see . + */ + +#include "libheif/heif.h" +#include "libheif/heif_plugin.h" +#include "decoder_jpeg.h" +#include +#include +#include +#include +#include + +extern "C" { +#include +} + + +struct jpeg_decoder +{ + std::vector data; +}; + +static const char kSuccess[] = "Success"; + +static const int JPEG_PLUGIN_PRIORITY = 100; + +#define MAX_PLUGIN_NAME_LENGTH 80 + +static char plugin_name[MAX_PLUGIN_NAME_LENGTH]; + +#define xstr(s) str(s) +#define str(s) #s + +static const char* jpeg_plugin_name() +{ +#ifdef LIBJPEG_TURBO_VERSION + snprintf(plugin_name, MAX_PLUGIN_NAME_LENGTH-1, "libjpeg-turbo " xstr(LIBJPEG_TURBO_VERSION) " (libjpeg %d.%d)", JPEG_LIB_VERSION/10, JPEG_LIB_VERSION%10); + plugin_name[MAX_PLUGIN_NAME_LENGTH-1] = 0; +#else + sprintf(plugin_name, "libjpeg %d.%d", JPEG_LIB_VERSION/10, JPEG_LIB_VERSION%10); +#endif + + return plugin_name; +} + + +static void jpeg_init_plugin() +{ +} + + +static void jpeg_deinit_plugin() +{ +} + + +static int jpeg_does_support_format(enum heif_compression_format format) +{ + if (format == heif_compression_JPEG) { + return JPEG_PLUGIN_PRIORITY; + } + else { + return 0; + } +} + + +struct heif_error jpeg_new_decoder(void** dec) +{ + struct jpeg_decoder* decoder = new jpeg_decoder(); + *dec = decoder; + + struct heif_error err = {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; + return err; +} + + +void jpeg_free_decoder(void* decoder_raw) +{ + struct jpeg_decoder* decoder = (jpeg_decoder*) decoder_raw; + + if (!decoder) { + return; + } + + delete decoder; +} + + +void jpeg_set_strict_decoding(void* decoder_raw, int flag) +{ +// struct jpeg_decoder* decoder = (jpeg_decoder*) decoder_raw; +} + + +struct heif_error jpeg_push_data(void* decoder_raw, const void* frame_data, size_t frame_size) +{ + struct jpeg_decoder* decoder = (struct jpeg_decoder*) decoder_raw; + + const uint8_t* input_data = (const uint8_t*)frame_data; + + decoder->data.insert(decoder->data.end(), input_data, input_data + frame_size); + + struct heif_error err = {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; + return err; +} + + +struct heif_error jpeg_decode_image(void* decoder_raw, struct heif_image** out_img) +{ + struct jpeg_decoder* decoder = (struct jpeg_decoder*) decoder_raw; + + + struct jpeg_decompress_struct cinfo; + struct jpeg_error_mgr jerr; + + // to store embedded icc profile +// uint32_t iccLen; +// uint8_t* iccBuffer = NULL; + +// std::vector xmpData; +// std::vector exifData; + + // initialize decompressor + + jpeg_create_decompress(&cinfo); + + cinfo.err = jpeg_std_error(&jerr); + jpeg_mem_src(&cinfo, decoder->data.data(), static_cast(decoder->data.size())); + + /* Adding this part to prepare for icc profile reading. */ +// jpeg_save_markers(&cinfo, JPEG_ICC_MARKER, 0xFFFF); +// jpeg_save_markers(&cinfo, JPEG_XMP_MARKER, 0xFFFF); +// jpeg_save_markers(&cinfo, JPEG_EXIF_MARKER, 0xFFFF); + + jpeg_read_header(&cinfo, TRUE); + +// bool embeddedIccFlag = ReadICCProfileFromJPEG(&cinfo, &iccBuffer, &iccLen); +// bool embeddedXMPFlag = ReadXMPFromJPEG(&cinfo, xmpData); +// if (embeddedXMPFlag) { +// img.xmp = xmpData; +// } + +// bool embeddedEXIFFlag = ReadEXIFFromJPEG(&cinfo, exifData); +// if (embeddedEXIFFlag) { +// img.exif = exifData; +// img.orientation = (heif_orientation) read_exif_orientation_tag(exifData.data(), (int) exifData.size()); +// } + + if (cinfo.jpeg_color_space == JCS_GRAYSCALE) { + cinfo.out_color_space = JCS_GRAYSCALE; + + jpeg_start_decompress(&cinfo); + + JSAMPARRAY buffer; + buffer = (*cinfo.mem->alloc_sarray) + ((j_common_ptr) &cinfo, JPOOL_IMAGE, cinfo.output_width * cinfo.output_components, 1); + + + // create destination image + + struct heif_image* heif_img = nullptr; + struct heif_error err = heif_image_create(cinfo.output_width, cinfo.output_height, + heif_colorspace_monochrome, + heif_chroma_monochrome, + &heif_img); + if (err.code != heif_error_Ok) { + assert(heif_img==nullptr); + return err; + } + + heif_image_add_plane(heif_img, heif_channel_Y, cinfo.output_width, cinfo.output_height, 8); + + int y_stride; + uint8_t* py = heif_image_get_plane(heif_img, heif_channel_Y, &y_stride); + + + // read the image + + while (cinfo.output_scanline < cinfo.output_height) { + (void) jpeg_read_scanlines(&cinfo, buffer, 1); + + memcpy(py + (cinfo.output_scanline - 1) * y_stride, *buffer, cinfo.output_width); + } + + *out_img = heif_img; + } + else { + cinfo.out_color_space = JCS_YCbCr; + + jpeg_start_decompress(&cinfo); + + JSAMPARRAY buffer; + buffer = (*cinfo.mem->alloc_sarray) + ((j_common_ptr) &cinfo, JPOOL_IMAGE, cinfo.output_width * cinfo.output_components, 1); + + + // create destination image + + struct heif_image* heif_img = nullptr; + struct heif_error err = heif_image_create(cinfo.output_width, cinfo.output_height, + heif_colorspace_YCbCr, + heif_chroma_420, + &heif_img); + if (err.code != heif_error_Ok) { + assert(heif_img==nullptr); + return err; + } + + heif_image_add_plane(heif_img, heif_channel_Y, cinfo.output_width, cinfo.output_height, 8); + heif_image_add_plane(heif_img, heif_channel_Cb, (cinfo.output_width + 1) / 2, (cinfo.output_height + 1) / 2, 8); + heif_image_add_plane(heif_img, heif_channel_Cr, (cinfo.output_width + 1) / 2, (cinfo.output_height + 1) / 2, 8); + + int y_stride; + int cb_stride; + int cr_stride; + uint8_t* py = heif_image_get_plane(heif_img, heif_channel_Y, &y_stride); + uint8_t* pcb = heif_image_get_plane(heif_img, heif_channel_Cb, &cb_stride); + uint8_t* pcr = heif_image_get_plane(heif_img, heif_channel_Cr, &cr_stride); + + // read the image + + //printf("jpeg size: %d %d\n",cinfo.output_width, cinfo.output_height); + + while (cinfo.output_scanline < cinfo.output_height) { + JOCTET* bufp; + + (void) jpeg_read_scanlines(&cinfo, buffer, 1); + + bufp = buffer[0]; + + int y = cinfo.output_scanline - 1; + + for (unsigned int x = 0; x < cinfo.output_width; x += 2) { + py[y * y_stride + x] = *bufp++; + pcb[y / 2 * cb_stride + x / 2] = *bufp++; + pcr[y / 2 * cr_stride + x / 2] = *bufp++; + + if (x + 1 < cinfo.output_width) { + py[y * y_stride + x + 1] = *bufp++; + } + + bufp += 2; + } + + + if (cinfo.output_scanline < cinfo.output_height) { + (void) jpeg_read_scanlines(&cinfo, buffer, 1); + + bufp = buffer[0]; + + y = cinfo.output_scanline - 1; + + for (unsigned int x = 0; x < cinfo.output_width; x++) { + py[y * y_stride + x] = *bufp++; + bufp += 2; + } + } + } + + *out_img = heif_img; + } + +// if (embeddedIccFlag && iccLen > 0) { +// heif_image_set_raw_color_profile(image, "prof", iccBuffer, (size_t) iccLen); +// } + + // cleanup +// free(iccBuffer); + jpeg_finish_decompress(&cinfo); + jpeg_destroy_decompress(&cinfo); + + decoder->data.clear(); + + return heif_error_ok; +} + + +static const struct heif_decoder_plugin decoder_jpeg + { + 3, + jpeg_plugin_name, + jpeg_init_plugin, + jpeg_deinit_plugin, + jpeg_does_support_format, + jpeg_new_decoder, + jpeg_free_decoder, + jpeg_push_data, + jpeg_decode_image, + jpeg_set_strict_decoding, + "jpeg" + }; + + +const struct heif_decoder_plugin* get_decoder_plugin_jpeg() +{ + return &decoder_jpeg; +} + + +#if PLUGIN_JPEG_DECODER +heif_plugin_info plugin_info { + 1, + heif_plugin_type_decoder, + &decoder_jpeg +}; +#endif diff --git a/libheif/plugins/decoder_openh264.h b/libheif/plugins/decoder_openh264.h new file mode 100644 index 0000000000..d40e441b15 --- /dev/null +++ b/libheif/plugins/decoder_openh264.h @@ -0,0 +1,34 @@ +/* + * HEIF codec. + * Copyright (c) 2023 Dirk Farin + * + * This file is part of libheif. + * + * libheif is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * libheif is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with libheif. If not, see . + */ + +#ifndef LIBHEIF_DECODER_JPEG_H +#define LIBHEIF_DECODER_JPEG_H + +#include "common_utils.h" + +const struct heif_decoder_plugin* get_decoder_plugin_jpeg(); + +#if PLUGIN_JPEG_DECODER +extern "C" { +MAYBE_UNUSED LIBHEIF_API extern heif_plugin_info plugin_info; +} +#endif + +#endif From 8f4463e01f40f58ddcab66cf931b20faa7e71a64 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 15:08:36 +0200 Subject: [PATCH 10/13] search for OpenH264 library --- CMakeLists.txt | 26 ++++++++++++++++++++++++++ cmake/modules/FindOpenH264.cmake | 26 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 cmake/modules/FindOpenH264.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3189f2c39a..9a6038e693 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,6 +161,23 @@ if (WITH_VVENC) endif() endif () +# openh264 decoder + +plugin_option(OPENH264_DECODER "OpenH264 decoder" ON OFF) +plugin_option(OPENH264_ENCODER "OpenH264 encoder" ON OFF) +if (WITH_OPENH264_ENCODER OR WITH_OPENH264_DECODER) + find_package(OpenH264) + + # When decoding/encoding is disabled, overwrite the *_FOUND variables, because they are used to ultimately decide what to build. + # TODO + if (OpenH264_FOUND AND WITH_OPENH264_DECODER) + set(OpenH264_DECODER_FOUND TRUE) + endif() + if (OpenH264_FOUND AND WITH_OPENH264_ENCODER) + set(OpenH264_ENCODER_FOUND TRUE) + endif() +endif() + # dav1d plugin_option(DAV1D "Dav1d AV1 decoder" OFF ON) @@ -248,6 +265,8 @@ plugin_compilation_info(SvtEnc SvtEnc "SVT AV1 encoder") plugin_compilation_info(RAV1E RAV1E "Rav1e AV1 encoder") plugin_compilation_info(JPEG_DECODER JPEG "JPEG decoder") plugin_compilation_info(JPEG_ENCODER JPEG "JPEG encoder") +plugin_compilation_info(OpenH264_DECODER OpenH264_DECODER "OpenH264 decoder") +plugin_compilation_info(OpenH264_ENCODER OpenH264_ENCODER "OpenH264 encoder") plugin_compilation_info(OpenJPEG_DECODER OpenJPEG "OpenJPEG J2K decoder") plugin_compilation_info(OpenJPEG_ENCODER OpenJPEG "OpenJPEG J2K encoder") # plugin_compilation_info(OPENJPH_DECODER OPENJPH "OpenJPH HT-J2K decoder") @@ -316,6 +335,12 @@ endif() if (vvdec_FOUND AND WITH_VVDEC) set(SUPPORTS_VVC_DECODING TRUE) endif() +if (OpenH264_DECODER_FOUND) + set(SUPPORTS_AVC_DECODING TRUE) +endif() +if (OpenH264_ENCODER_FOUND) + set(SUPPORTS_AVC_ENCODING TRUE) +endif() if (WITH_UNCOMPRESSED_CODEC) set(SUPPORTS_UNCOMPRESSED_DECODING TRUE) @@ -327,6 +352,7 @@ message("format decoding encoding") format_compilation_info("HEIC" SUPPORTS_HEIC_DECODING SUPPORTS_HEIC_ENCODING) format_compilation_info("AVIF" SUPPORTS_AVIF_DECODING SUPPORTS_AVIF_ENCODING) format_compilation_info("VVC" SUPPORTS_VVC_DECODING SUPPORTS_VVC_ENCODING) +format_compilation_info("AVC" SUPPORTS_AVC_DECODING SUPPORTS_AVC_ENCODING) format_compilation_info("JPEG" SUPPORTS_JPEG_DECODING SUPPORTS_JPEG_ENCODING) format_compilation_info("JPEG2000" SUPPORTS_J2K_DECODING SUPPORTS_J2K_ENCODING) format_compilation_info("JPEG2000-HT" SUPPORTS_J2K_HT_DECODING SUPPORTS_J2K_HT_ENCODING) diff --git a/cmake/modules/FindOpenH264.cmake b/cmake/modules/FindOpenH264.cmake new file mode 100644 index 0000000000..a4933249ea --- /dev/null +++ b/cmake/modules/FindOpenH264.cmake @@ -0,0 +1,26 @@ +include(LibFindMacros) +include(CheckSymbolExists) + +libfind_pkg_check_modules(OpenH264_PKGCONF openh264) + +find_path(OpenH264_INCLUDE_DIR + NAMES wels/codec_api.h + HINTS ${OpenH264_PKGCONF_INCLUDE_DIRS} ${OpenH264_PKGCONF_INCLUDEDIR} + PATH_SUFFIXES OpenH264 +) + +find_library(OpenH264_LIBRARY + NAMES libopenh264 openh264 + HINTS ${OpenH264_PKGCONF_LIBRARY_DIRS} ${OpenH264_PKGCONF_LIBDIR} +) + +set(OpenH264_PROCESS_LIBS OpenH264_LIBRARY) +set(OpenH264_PROCESS_INCLUDES OpenH264_INCLUDE_DIR) +libfind_process(OpenH264) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(OpenH264 + REQUIRED_VARS + OpenH264_INCLUDE_DIR + OpenH264_LIBRARIES +) From 0bdc85813933eeaa90ad5e8642b3d0a948ffe6a6 Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 23:26:53 +0200 Subject: [PATCH 11/13] implement AVC decoder based on OpenH264 --- CMakeLists.txt | 12 +- examples/heif_dec.cc | 7 +- libheif/codecs/avc.cc | 14 +- libheif/codecs/avc.h | 2 +- libheif/plugin_registry.cc | 8 + libheif/plugins/CMakeLists.txt | 11 +- libheif/plugins/decoder_openh264.cc | 335 ++++++++++++++-------------- libheif/plugins/decoder_openh264.h | 10 +- 8 files changed, 219 insertions(+), 180 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a6038e693..9c008b84a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,21 +163,23 @@ endif () # openh264 decoder -plugin_option(OPENH264_DECODER "OpenH264 decoder" ON OFF) -plugin_option(OPENH264_ENCODER "OpenH264 encoder" ON OFF) -if (WITH_OPENH264_ENCODER OR WITH_OPENH264_DECODER) +plugin_option(OpenH264_DECODER "OpenH264 decoder" ON OFF) +plugin_option(OpenH264_ENCODER "OpenH264 encoder" ON OFF) +if (WITH_OpenH264_ENCODER OR WITH_OpenH264_DECODER) find_package(OpenH264) # When decoding/encoding is disabled, overwrite the *_FOUND variables, because they are used to ultimately decide what to build. # TODO - if (OpenH264_FOUND AND WITH_OPENH264_DECODER) + if (OpenH264_FOUND AND WITH_OpenH264_DECODER) set(OpenH264_DECODER_FOUND TRUE) endif() - if (OpenH264_FOUND AND WITH_OPENH264_ENCODER) + if (OpenH264_FOUND AND WITH_OpenH264_ENCODER) set(OpenH264_ENCODER_FOUND TRUE) endif() endif() + + # dav1d plugin_option(DAV1D "Dav1d AV1 decoder" OFF ON) diff --git a/examples/heif_dec.cc b/examples/heif_dec.cc index 9742666b33..5e2b9cb213 100644 --- a/examples/heif_dec.cc +++ b/examples/heif_dec.cc @@ -171,11 +171,14 @@ void list_all_decoders() std::cout << "HEIC decoders:\n"; list_decoders(heif_compression_HEVC); + std::cout << "AVIF decoders:\n"; + list_decoders(heif_compression_AV1); + std::cout << "VVIC decoders:\n"; list_decoders(heif_compression_VVC); - std::cout << "AVIF decoders:\n"; - list_decoders(heif_compression_AV1); + std::cout << "AVC decoders:\n"; + list_decoders(heif_compression_AVC); std::cout << "JPEG decoders:\n"; list_decoders(heif_compression_JPEG); diff --git a/libheif/codecs/avc.cc b/libheif/codecs/avc.cc index a6cbfdb165..18218e979e 100644 --- a/libheif/codecs/avc.cc +++ b/libheif/codecs/avc.cc @@ -147,13 +147,23 @@ std::string Box_avcC::profileIndicationAsText() const { } -void Box_avcC::get_headers(std::vector& data) const +void Box_avcC::get_header_nals(std::vector& data) const { for (const auto& sps : m_sps) { + data.push_back((sps.size() >> 24) & 0xFF); + data.push_back((sps.size() >> 16) & 0xFF); + data.push_back((sps.size() >> 8) & 0xFF); + data.push_back((sps.size() >> 0) & 0xFF); + data.insert(data.end(), sps.begin(), sps.end()); } for (const auto& pps : m_pps) { + data.push_back((pps.size() >> 24) & 0xFF); + data.push_back((pps.size() >> 16) & 0xFF); + data.push_back((pps.size() >> 8) & 0xFF); + data.push_back((pps.size() >> 0) & 0xFF); + data.insert(data.end(), pps.begin(), pps.end()); } } @@ -259,7 +269,7 @@ Result> ImageItem_AVC::read_bitstream_configuration_data(he heif_suberror_No_av1C_box}; } - avcC_box->get_headers(data); + avcC_box->get_header_nals(data); return data; } diff --git a/libheif/codecs/avc.h b/libheif/codecs/avc.h index bfb8a1bc86..0233e7efe6 100644 --- a/libheif/codecs/avc.h +++ b/libheif/codecs/avc.h @@ -63,7 +63,7 @@ class Box_avcC : public Box { return m_pps; } - void get_headers(std::vector& data) const; + void get_header_nals(std::vector& data) const; std::string dump(Indent &) const override; diff --git a/libheif/plugin_registry.cc b/libheif/plugin_registry.cc index 38e4594f30..9bc196f467 100644 --- a/libheif/plugin_registry.cc +++ b/libheif/plugin_registry.cc @@ -86,6 +86,10 @@ #include "plugins/encoder_jpeg.h" #endif +#if HAVE_OpenH264_DECODER +#include "plugins/decoder_openh264.h" +#endif + #if HAVE_OPENJPEG_ENCODER #include "plugins/encoder_openjpeg.h" #endif @@ -203,6 +207,10 @@ void register_default_plugins() register_encoder(get_encoder_plugin_openjph()); #endif +#if HAVE_OpenH264_DECODER + register_decoder(get_decoder_plugin_openh264()); +#endif + #if WITH_UNCOMPRESSED_CODEC register_encoder(get_encoder_plugin_uncompressed()); #endif diff --git a/libheif/plugins/CMakeLists.txt b/libheif/plugins/CMakeLists.txt index 01564808dd..ac11cee405 100644 --- a/libheif/plugins/CMakeLists.txt +++ b/libheif/plugins/CMakeLists.txt @@ -108,6 +108,16 @@ set(VVENC_sources encoder_vvenc.cc encoder_vvenc.h) set(VVENC_extra_plugin_sources) plugin_compilation(vvenc vvenc vvenc_FOUND VVENC VVENC) +set(OpenH264_DECODER_sources decoder_openh264.cc decoder_openh264.h) +set(OpenH264_DECODER_extra_plugin_sources) +plugin_compilation(openh264dec OpenH264 OpenH264_DECODER_FOUND OpenH264_DECODER OpenH264_DECODER) + +message("a ${WITH_OpenH264_DECODER_PLUGIN}") +message("b ${WITH_OPENH264_DECODER_PLUGIN}") + +message("aa ${WITH_OpenH264_DECODER}") +message("bb ${WITH_OPENH264_DECODER}") + target_sources(heif PRIVATE encoder_mask.h encoder_mask.cc @@ -119,4 +129,3 @@ if (WITH_UNCOMPRESSED_CODEC) encoder_uncompressed.h encoder_uncompressed.cc) endif () - diff --git a/libheif/plugins/decoder_openh264.cc b/libheif/plugins/decoder_openh264.cc index 901bf92214..350099e961 100644 --- a/libheif/plugins/decoder_openh264.cc +++ b/libheif/plugins/decoder_openh264.cc @@ -1,6 +1,6 @@ /* - * JPEG codec. - * Copyright (c) 2023 Dirk Farin + * openh264 codec. + * Copyright (c) 2024 Dirk Farin * * This file is part of libheif. * @@ -20,61 +20,56 @@ #include "libheif/heif.h" #include "libheif/heif_plugin.h" -#include "decoder_jpeg.h" +#include "decoder_openh264.h" #include #include #include #include #include -extern "C" { -#include -} +#include -struct jpeg_decoder +struct openh264_decoder { std::vector data; }; static const char kSuccess[] = "Success"; -static const int JPEG_PLUGIN_PRIORITY = 100; +static const int OpenH264_PLUGIN_PRIORITY = 100; #define MAX_PLUGIN_NAME_LENGTH 80 static char plugin_name[MAX_PLUGIN_NAME_LENGTH]; -#define xstr(s) str(s) -#define str(s) #s +static heif_error kError_EOF = {heif_error_Decoder_plugin_error, heif_suberror_End_of_data, "Insufficient input data"}; + -static const char* jpeg_plugin_name() +static const char* openh264_plugin_name() { -#ifdef LIBJPEG_TURBO_VERSION - snprintf(plugin_name, MAX_PLUGIN_NAME_LENGTH-1, "libjpeg-turbo " xstr(LIBJPEG_TURBO_VERSION) " (libjpeg %d.%d)", JPEG_LIB_VERSION/10, JPEG_LIB_VERSION%10); - plugin_name[MAX_PLUGIN_NAME_LENGTH-1] = 0; -#else - sprintf(plugin_name, "libjpeg %d.%d", JPEG_LIB_VERSION/10, JPEG_LIB_VERSION%10); -#endif + OpenH264Version version = WelsGetCodecVersion(); + + sprintf(plugin_name, "OpenH264 %d.%d.%d", version.uMajor, version.uMinor, version.uRevision); return plugin_name; } -static void jpeg_init_plugin() +static void openh264_init_plugin() { } -static void jpeg_deinit_plugin() +static void openh264_deinit_plugin() { } -static int jpeg_does_support_format(enum heif_compression_format format) +static int openh264_does_support_format(enum heif_compression_format format) { - if (format == heif_compression_JPEG) { - return JPEG_PLUGIN_PRIORITY; + if (format == heif_compression_AVC) { + return OpenH264_PLUGIN_PRIORITY; } else { return 0; @@ -82,9 +77,9 @@ static int jpeg_does_support_format(enum heif_compression_format format) } -struct heif_error jpeg_new_decoder(void** dec) +struct heif_error openh264_new_decoder(void** dec) { - struct jpeg_decoder* decoder = new jpeg_decoder(); + struct openh264_decoder* decoder = new openh264_decoder(); *dec = decoder; struct heif_error err = {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; @@ -92,9 +87,9 @@ struct heif_error jpeg_new_decoder(void** dec) } -void jpeg_free_decoder(void* decoder_raw) +void openh264_free_decoder(void* decoder_raw) { - struct jpeg_decoder* decoder = (jpeg_decoder*) decoder_raw; + struct openh264_decoder* decoder = (openh264_decoder*) decoder_raw; if (!decoder) { return; @@ -104,17 +99,17 @@ void jpeg_free_decoder(void* decoder_raw) } -void jpeg_set_strict_decoding(void* decoder_raw, int flag) +void openh264_set_strict_decoding(void* decoder_raw, int flag) { -// struct jpeg_decoder* decoder = (jpeg_decoder*) decoder_raw; +// auto* decoder = (openh264_decoder*) decoder_raw; } -struct heif_error jpeg_push_data(void* decoder_raw, const void* frame_data, size_t frame_size) +struct heif_error openh264_push_data(void* decoder_raw, const void* frame_data, size_t frame_size) { - struct jpeg_decoder* decoder = (struct jpeg_decoder*) decoder_raw; + auto* decoder = (struct openh264_decoder*) decoder_raw; - const uint8_t* input_data = (const uint8_t*)frame_data; + const auto* input_data = (const uint8_t*) frame_data; decoder->data.insert(decoder->data.end(), input_data, input_data + frame_size); @@ -123,110 +118,153 @@ struct heif_error jpeg_push_data(void* decoder_raw, const void* frame_data, size } -struct heif_error jpeg_decode_image(void* decoder_raw, struct heif_image** out_img) +struct heif_error openh264_decode_image(void* decoder_raw, struct heif_image** out_img) { - struct jpeg_decoder* decoder = (struct jpeg_decoder*) decoder_raw; + auto* decoder = (struct openh264_decoder*) decoder_raw; + if (decoder->data.size() < 4) { + return kError_EOF; + } - struct jpeg_decompress_struct cinfo; - struct jpeg_error_mgr jerr; + const std::vector& indata = decoder->data; + std::vector scdata; - // to store embedded icc profile -// uint32_t iccLen; -// uint8_t* iccBuffer = NULL; + size_t idx = 0; + while (idx < indata.size()) { + if (indata.size() - 4 < idx) { + return kError_EOF; + } -// std::vector xmpData; -// std::vector exifData; + uint32_t size = ((indata[idx] << 24) | (indata[idx + 1] << 16) | (indata[idx + 2] << 8) | indata[idx + 3]); + idx += 4; - // initialize decompressor + if (indata.size() < size || indata.size() - size < idx) { + return kError_EOF; + } - jpeg_create_decompress(&cinfo); + scdata.push_back(0); + scdata.push_back(0); + scdata.push_back(1); - cinfo.err = jpeg_std_error(&jerr); - jpeg_mem_src(&cinfo, decoder->data.data(), static_cast(decoder->data.size())); + // check for need of start code emulation prevention - /* Adding this part to prepare for icc profile reading. */ -// jpeg_save_markers(&cinfo, JPEG_ICC_MARKER, 0xFFFF); -// jpeg_save_markers(&cinfo, JPEG_XMP_MARKER, 0xFFFF); -// jpeg_save_markers(&cinfo, JPEG_EXIF_MARKER, 0xFFFF); + bool do_start_code_emulation_check = true; - jpeg_read_header(&cinfo, TRUE); + while (do_start_code_emulation_check && size >= 3) { -// bool embeddedIccFlag = ReadICCProfileFromJPEG(&cinfo, &iccBuffer, &iccLen); -// bool embeddedXMPFlag = ReadXMPFromJPEG(&cinfo, xmpData); -// if (embeddedXMPFlag) { -// img.xmp = xmpData; -// } + bool found_start_code_emulation = false; -// bool embeddedEXIFFlag = ReadEXIFFromJPEG(&cinfo, exifData); -// if (embeddedEXIFFlag) { -// img.exif = exifData; -// img.orientation = (heif_orientation) read_exif_orientation_tag(exifData.data(), (int) exifData.size()); -// } + for (size_t i = 0; i < size - 3; i++) { + if (indata[idx + 0] == 0 && + indata[idx + 1] == 0 && + (indata[idx + 2] >= 0 && indata[idx + 2] <= 3)) { + scdata.push_back(0); + scdata.push_back(0); + scdata.push_back(3); - if (cinfo.jpeg_color_space == JCS_GRAYSCALE) { - cinfo.out_color_space = JCS_GRAYSCALE; + scdata.insert(scdata.end(), &indata[idx + 2], &indata[idx + i + 2]); + idx += i + 2; + size -= (uint32_t)(i + 2); + found_start_code_emulation = true; + break; + } + } - jpeg_start_decompress(&cinfo); + do_start_code_emulation_check = found_start_code_emulation; + } - JSAMPARRAY buffer; - buffer = (*cinfo.mem->alloc_sarray) - ((j_common_ptr) &cinfo, JPOOL_IMAGE, cinfo.output_width * cinfo.output_components, 1); + assert(size > 0); + scdata.insert(scdata.end(), &indata[idx], &indata[idx + size]); + idx += size; + } - // create destination image + if (idx != indata.size()) { + return kError_EOF; + } - struct heif_image* heif_img = nullptr; - struct heif_error err = heif_image_create(cinfo.output_width, cinfo.output_height, - heif_colorspace_monochrome, - heif_chroma_monochrome, - &heif_img); - if (err.code != heif_error_Ok) { - assert(heif_img==nullptr); - return err; - } + // input: encoded bitstream start position; should include start code prefix + unsigned char* pBuf = scdata.data(); - heif_image_add_plane(heif_img, heif_channel_Y, cinfo.output_width, cinfo.output_height, 8); + // input: encoded bit stream length; should include the size of start code prefix + int iSize = static_cast(scdata.size()); - int y_stride; - uint8_t* py = heif_image_get_plane(heif_img, heif_channel_Y, &y_stride); + //output: [0~2] for Y,U,V buffer for Decoding only + unsigned char* pData[3] = {nullptr, nullptr, nullptr}; + // in-out: for Decoding only: declare and initialize the output buffer info, this should never co-exist with Parsing only - // read the image + SBufferInfo sDstBufInfo; + memset(&sDstBufInfo, 0, sizeof(SBufferInfo)); - while (cinfo.output_scanline < cinfo.output_height) { - (void) jpeg_read_scanlines(&cinfo, buffer, 1); + // Step 2:decoder creation + ISVCDecoder* pSvcDecoder; + WelsCreateDecoder(&pSvcDecoder); + if (!pSvcDecoder) { + return {heif_error_Decoder_plugin_error, + heif_suberror_Unspecified, + "Cannot create OpenH264 decoder"}; + } - memcpy(py + (cinfo.output_scanline - 1) * y_stride, *buffer, cinfo.output_width); - } + std::unique_ptr dummy_h264_decoder_ptr(pSvcDecoder, WelsDestroyDecoder); - *out_img = heif_img; + + // Step 3:declare required parameter, used to differentiate Decoding only and Parsing only + SDecodingParam sDecParam = {0}; + sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; + + //for Parsing only, the assignment is mandatory + // sDecParam.bParseOnly = true; + + // Step 4:initialize the parameter and decoder context, allocate memory + pSvcDecoder->Initialize(&sDecParam); + + // Step 5:do actual decoding process in slice level; this can be done in a loop until data ends + + //for Decoding only + int iRet = pSvcDecoder->DecodeFrameNoDelay(pBuf, iSize, pData, &sDstBufInfo); + + if (iRet != 0) { + return {heif_error_Decoder_plugin_error, + heif_suberror_Unspecified, + "OpenH264 decoder error"}; } - else { - cinfo.out_color_space = JCS_YCbCr; - jpeg_start_decompress(&cinfo); + /* + // TODO: I receive an iBufferStatus==0, but the output image is still decoded + if (sDstBufInfo.iBufferStatus == 0) { + return {heif_error_Decoder_plugin_error, + heif_suberror_Unspecified, + "OpenH264 decoder did not output any image"}; + } + */ + + uint32_t width = sDstBufInfo.UsrData.sSystemBuffer.iWidth; + uint32_t height = sDstBufInfo.UsrData.sSystemBuffer.iHeight; - JSAMPARRAY buffer; - buffer = (*cinfo.mem->alloc_sarray) - ((j_common_ptr) &cinfo, JPOOL_IMAGE, cinfo.output_width * cinfo.output_components, 1); + struct heif_image* heif_img; + struct heif_error err{}; + uint32_t cwidth, cheight; - // create destination image + if (sDstBufInfo.UsrData.sSystemBuffer.iFormat == videoFormatI420) { + cwidth = (width + 1) / 2; + cheight = (height + 1) / 2; - struct heif_image* heif_img = nullptr; - struct heif_error err = heif_image_create(cinfo.output_width, cinfo.output_height, - heif_colorspace_YCbCr, - heif_chroma_420, - &heif_img); + err = heif_image_create(width, height, + heif_colorspace_YCbCr, + heif_chroma_420, + &heif_img); if (err.code != heif_error_Ok) { - assert(heif_img==nullptr); + assert(heif_img == nullptr); return err; } - heif_image_add_plane(heif_img, heif_channel_Y, cinfo.output_width, cinfo.output_height, 8); - heif_image_add_plane(heif_img, heif_channel_Cb, (cinfo.output_width + 1) / 2, (cinfo.output_height + 1) / 2, 8); - heif_image_add_plane(heif_img, heif_channel_Cr, (cinfo.output_width + 1) / 2, (cinfo.output_height + 1) / 2, 8); + *out_img = heif_img; + + heif_image_add_plane(heif_img, heif_channel_Y, width, height, 8); + heif_image_add_plane(heif_img, heif_channel_Cb, cwidth, cheight, 8); + heif_image_add_plane(heif_img, heif_channel_Cr, cwidth, cheight, 8); int y_stride; int cb_stride; @@ -235,57 +273,27 @@ struct heif_error jpeg_decode_image(void* decoder_raw, struct heif_image** out_i uint8_t* pcb = heif_image_get_plane(heif_img, heif_channel_Cb, &cb_stride); uint8_t* pcr = heif_image_get_plane(heif_img, heif_channel_Cr, &cr_stride); - // read the image - - //printf("jpeg size: %d %d\n",cinfo.output_width, cinfo.output_height); - - while (cinfo.output_scanline < cinfo.output_height) { - JOCTET* bufp; - - (void) jpeg_read_scanlines(&cinfo, buffer, 1); - - bufp = buffer[0]; - - int y = cinfo.output_scanline - 1; - - for (unsigned int x = 0; x < cinfo.output_width; x += 2) { - py[y * y_stride + x] = *bufp++; - pcb[y / 2 * cb_stride + x / 2] = *bufp++; - pcr[y / 2 * cr_stride + x / 2] = *bufp++; - - if (x + 1 < cinfo.output_width) { - py[y * y_stride + x + 1] = *bufp++; - } - - bufp += 2; - } + int ystride = sDstBufInfo.UsrData.sSystemBuffer.iStride[0]; + int cstride = sDstBufInfo.UsrData.sSystemBuffer.iStride[1]; - - if (cinfo.output_scanline < cinfo.output_height) { - (void) jpeg_read_scanlines(&cinfo, buffer, 1); - - bufp = buffer[0]; - - y = cinfo.output_scanline - 1; - - for (unsigned int x = 0; x < cinfo.output_width; x++) { - py[y * y_stride + x] = *bufp++; - bufp += 2; - } - } + for (uint32_t y = 0; y < height; y++) { + memcpy(py + y * y_stride, sDstBufInfo.pDst[0] + y * ystride, width); } - *out_img = heif_img; + for (uint32_t y = 0; y < (height + 1) / 2; y++) { + memcpy(pcb + y * cb_stride, sDstBufInfo.pDst[1] + y * cstride, (width + 1) / 2); + memcpy(pcr + y * cr_stride, sDstBufInfo.pDst[2] + y * cstride, (width + 1) / 2); + } + } + else { + return {heif_error_Decoder_plugin_error, + heif_suberror_Unspecified, + "Unsupported image pixel format"}; } -// if (embeddedIccFlag && iccLen > 0) { -// heif_image_set_raw_color_profile(image, "prof", iccBuffer, (size_t) iccLen); -// } + // Step 6:uninitialize the decoder and memory free - // cleanup -// free(iccBuffer); - jpeg_finish_decompress(&cinfo); - jpeg_destroy_decompress(&cinfo); + pSvcDecoder->Uninitialize(); // TODO: do we have to Uninitialize when an error is returned? decoder->data.clear(); @@ -293,32 +301,31 @@ struct heif_error jpeg_decode_image(void* decoder_raw, struct heif_image** out_i } -static const struct heif_decoder_plugin decoder_jpeg - { +static const struct heif_decoder_plugin decoder_openh264{ 3, - jpeg_plugin_name, - jpeg_init_plugin, - jpeg_deinit_plugin, - jpeg_does_support_format, - jpeg_new_decoder, - jpeg_free_decoder, - jpeg_push_data, - jpeg_decode_image, - jpeg_set_strict_decoding, - "jpeg" - }; - - -const struct heif_decoder_plugin* get_decoder_plugin_jpeg() + openh264_plugin_name, + openh264_init_plugin, + openh264_deinit_plugin, + openh264_does_support_format, + openh264_new_decoder, + openh264_free_decoder, + openh264_push_data, + openh264_decode_image, + openh264_set_strict_decoding, + "openh264" +}; + + +const struct heif_decoder_plugin* get_decoder_plugin_openh264() { - return &decoder_jpeg; + return &decoder_openh264; } -#if PLUGIN_JPEG_DECODER +#if PLUGIN_OpenH264_DECODER heif_plugin_info plugin_info { 1, heif_plugin_type_decoder, - &decoder_jpeg + &decoder_openh264 }; #endif diff --git a/libheif/plugins/decoder_openh264.h b/libheif/plugins/decoder_openh264.h index d40e441b15..7289b6b2fd 100644 --- a/libheif/plugins/decoder_openh264.h +++ b/libheif/plugins/decoder_openh264.h @@ -1,6 +1,6 @@ /* * HEIF codec. - * Copyright (c) 2023 Dirk Farin + * Copyright (c) 2024 Dirk Farin * * This file is part of libheif. * @@ -18,14 +18,14 @@ * along with libheif. If not, see . */ -#ifndef LIBHEIF_DECODER_JPEG_H -#define LIBHEIF_DECODER_JPEG_H +#ifndef LIBHEIF_DECODER_OPENH264_H +#define LIBHEIF_DECODER_OPENH264_H #include "common_utils.h" -const struct heif_decoder_plugin* get_decoder_plugin_jpeg(); +const struct heif_decoder_plugin* get_decoder_plugin_openh264(); -#if PLUGIN_JPEG_DECODER +#if PLUGIN_OpenH264_DECODER extern "C" { MAYBE_UNUSED LIBHEIF_API extern heif_plugin_info plugin_info; } From 9691be81ae98d09c61c1f001e899f2edeb8b363f Mon Sep 17 00:00:00 2001 From: Dirk Farin Date: Sun, 1 Sep 2024 23:30:41 +0200 Subject: [PATCH 12/13] cleanup --- libheif/plugins/CMakeLists.txt | 6 ------ libheif/plugins/decoder_openh264.cc | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/libheif/plugins/CMakeLists.txt b/libheif/plugins/CMakeLists.txt index ac11cee405..58418d9983 100644 --- a/libheif/plugins/CMakeLists.txt +++ b/libheif/plugins/CMakeLists.txt @@ -112,12 +112,6 @@ set(OpenH264_DECODER_sources decoder_openh264.cc decoder_openh264.h) set(OpenH264_DECODER_extra_plugin_sources) plugin_compilation(openh264dec OpenH264 OpenH264_DECODER_FOUND OpenH264_DECODER OpenH264_DECODER) -message("a ${WITH_OpenH264_DECODER_PLUGIN}") -message("b ${WITH_OPENH264_DECODER_PLUGIN}") - -message("aa ${WITH_OpenH264_DECODER}") -message("bb ${WITH_OPENH264_DECODER}") - target_sources(heif PRIVATE encoder_mask.h encoder_mask.cc diff --git a/libheif/plugins/decoder_openh264.cc b/libheif/plugins/decoder_openh264.cc index 350099e961..a3eb6c60ca 100644 --- a/libheif/plugins/decoder_openh264.cc +++ b/libheif/plugins/decoder_openh264.cc @@ -79,7 +79,7 @@ static int openh264_does_support_format(enum heif_compression_format format) struct heif_error openh264_new_decoder(void** dec) { - struct openh264_decoder* decoder = new openh264_decoder(); + auto* decoder = new openh264_decoder(); *dec = decoder; struct heif_error err = {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; @@ -89,7 +89,7 @@ struct heif_error openh264_new_decoder(void** dec) void openh264_free_decoder(void* decoder_raw) { - struct openh264_decoder* decoder = (openh264_decoder*) decoder_raw; + auto* decoder = (openh264_decoder*) decoder_raw; if (!decoder) { return; @@ -210,7 +210,7 @@ struct heif_error openh264_decode_image(void* decoder_raw, struct heif_image** o // Step 3:declare required parameter, used to differentiate Decoding only and Parsing only - SDecodingParam sDecParam = {0}; + SDecodingParam sDecParam{}; sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; //for Parsing only, the assignment is mandatory From 5cdb7c2cb4b56bb21b2034c0b5f87f44aef6016d Mon Sep 17 00:00:00 2001 From: Brad Hards Date: Mon, 2 Sep 2024 20:55:56 +1000 Subject: [PATCH 13/13] nalu: fix clang tidy warning for trivial destructor --- libheif/plugins/nalu_utils.cc | 5 ----- libheif/plugins/nalu_utils.h | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/libheif/plugins/nalu_utils.cc b/libheif/plugins/nalu_utils.cc index 7a8495c81e..f4f3731d60 100644 --- a/libheif/plugins/nalu_utils.cc +++ b/libheif/plugins/nalu_utils.cc @@ -30,11 +30,6 @@ NalUnit::NalUnit() nal_data_size = 0; } -NalUnit::~NalUnit() -{ - -} - bool NalUnit::set_data(const unsigned char* in_data, int n) { nal_data_ptr = in_data; diff --git a/libheif/plugins/nalu_utils.h b/libheif/plugins/nalu_utils.h index 2ed75d073c..7c4b86f7ad 100644 --- a/libheif/plugins/nalu_utils.h +++ b/libheif/plugins/nalu_utils.h @@ -32,7 +32,7 @@ class NalUnit { public: NalUnit(); - ~NalUnit(); + ~NalUnit() = default; bool set_data(const unsigned char* in_data, int n); int size() const { return nal_data_size; } int unit_type() const { return nal_unit_type; }