plugins: refactor NALU parsing for shared future usage

CMakeLists.txt

@@ -161,6 +161,25 @@ if (WITH_VVENC)
  endif()
 endif ()
 
+# openh264 decoder
+
+plugin_option(OpenH264_DECODER "OpenH264 decoder" ON OFF)
+plugin_option(OpenH264_ENCODER "OpenH264 encoder" ON OFF)
+if (WITH_OpenH264_ENCODER OR WITH_OpenH264_DECODER)
+ find_package(OpenH264)
+
+ # When decoding/encoding is disabled, overwrite the *_FOUND variables, because they are used to ultimately decide what to build.
+ # TODO
+ if (OpenH264_FOUND AND WITH_OpenH264_DECODER)
+ set(OpenH264_DECODER_FOUND TRUE)
+ endif()
+ if (OpenH264_FOUND AND WITH_OpenH264_ENCODER)
+ set(OpenH264_ENCODER_FOUND TRUE)
+ endif()
+endif()
+
+
+
 # dav1d
 
 plugin_option(DAV1D "Dav1d AV1 decoder" OFF ON)
@@ -248,6 +267,8 @@ plugin_compilation_info(SvtEnc SvtEnc "SVT AV1 encoder")
 plugin_compilation_info(RAV1E RAV1E "Rav1e AV1 encoder")
 plugin_compilation_info(JPEG_DECODER JPEG "JPEG decoder")
 plugin_compilation_info(JPEG_ENCODER JPEG "JPEG encoder")
+plugin_compilation_info(OpenH264_DECODER OpenH264_DECODER "OpenH264 decoder")
+plugin_compilation_info(OpenH264_ENCODER OpenH264_ENCODER "OpenH264 encoder")
 plugin_compilation_info(OpenJPEG_DECODER OpenJPEG "OpenJPEG J2K decoder")
 plugin_compilation_info(OpenJPEG_ENCODER OpenJPEG "OpenJPEG J2K encoder")
 # plugin_compilation_info(OPENJPH_DECODER OPENJPH "OpenJPH HT-J2K decoder")
@@ -316,6 +337,12 @@ endif()
 if (vvdec_FOUND AND WITH_VVDEC)
  set(SUPPORTS_VVC_DECODING TRUE)
 endif()
+if (OpenH264_DECODER_FOUND)
+ set(SUPPORTS_AVC_DECODING TRUE)
+endif()
+if (OpenH264_ENCODER_FOUND)
+ set(SUPPORTS_AVC_ENCODING TRUE)
+endif()
 
 if (WITH_UNCOMPRESSED_CODEC)
  set(SUPPORTS_UNCOMPRESSED_DECODING TRUE)
@@ -327,6 +354,7 @@ message("format decoding encoding")
 format_compilation_info("HEIC" SUPPORTS_HEIC_DECODING SUPPORTS_HEIC_ENCODING)
 format_compilation_info("AVIF" SUPPORTS_AVIF_DECODING SUPPORTS_AVIF_ENCODING)
 format_compilation_info("VVC" SUPPORTS_VVC_DECODING SUPPORTS_VVC_ENCODING)
+format_compilation_info("AVC" SUPPORTS_AVC_DECODING SUPPORTS_AVC_ENCODING)
 format_compilation_info("JPEG" SUPPORTS_JPEG_DECODING SUPPORTS_JPEG_ENCODING)
 format_compilation_info("JPEG2000" SUPPORTS_J2K_DECODING SUPPORTS_J2K_ENCODING)
 format_compilation_info("JPEG2000-HT" SUPPORTS_J2K_HT_DECODING SUPPORTS_J2K_HT_ENCODING)

cmake/modules/FindOpenH264.cmake

@@ -0,0 +1,26 @@
+include(LibFindMacros)
+include(CheckSymbolExists)
+
+libfind_pkg_check_modules(OpenH264_PKGCONF openh264)
+
+find_path(OpenH264_INCLUDE_DIR
+ NAMES wels/codec_api.h
+ HINTS ${OpenH264_PKGCONF_INCLUDE_DIRS} ${OpenH264_PKGCONF_INCLUDEDIR}
+ PATH_SUFFIXES OpenH264
+)
+
+find_library(OpenH264_LIBRARY
+ NAMES libopenh264 openh264
+ HINTS ${OpenH264_PKGCONF_LIBRARY_DIRS} ${OpenH264_PKGCONF_LIBDIR}
+)
+
+set(OpenH264_PROCESS_LIBS OpenH264_LIBRARY)
+set(OpenH264_PROCESS_INCLUDES OpenH264_INCLUDE_DIR)
+libfind_process(OpenH264)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OpenH264
+ REQUIRED_VARS
+ OpenH264_INCLUDE_DIR
+ OpenH264_LIBRARIES
+)

examples/heif_dec.cc

@@ -171,11 +171,14 @@ void list_all_decoders()
  std::cout << "HEIC decoders:\n";
  list_decoders(heif_compression_HEVC);
 
+ std::cout << "AVIF decoders:\n";
+ list_decoders(heif_compression_AV1);
+
  std::cout << "VVIC decoders:\n";
  list_decoders(heif_compression_VVC);
 
- std::cout << "AVIF decoders:\n";
- list_decoders(heif_compression_AV1);
+ std::cout << "AVC decoders:\n";
+ list_decoders(heif_compression_AVC);
 
  std::cout << "JPEG decoders:\n";
  list_decoders(heif_compression_JPEG);

go/heif/heif.go

@@ -248,6 +248,8 @@ const (
 
  SuberrorNoAV1CBox = C.heif_suberror_No_av1C_box
 
+ SuberrorNoAVCCBox = C.heif_suberror_No_avcC_box
+
  SuberrorInvalidCleanAperture = C.heif_suberror_Invalid_clean_aperture
 
  // Invalid specification of overlay image

libheif/api/libheif/heif.h

@@ -246,6 +246,8 @@ enum heif_suberror_code
  // icbr is only needed in some situations, this error is for those cases
  heif_suberror_No_icbr_box = 142,
 
+ heif_suberror_No_avcC_box = 143,
+
  // Decompressing generic compression or header compression data failed (e.g. bitstream corruption)
  heif_suberror_Decompression_invalid_data = 150,
 

libheif/api/libheif/heif_emscripten.h

@@ -359,6 +359,7 @@ EMSCRIPTEN_BINDINGS(libheif) {
  .value("heif_suberror_Invalid_grid_data", heif_suberror_Invalid_grid_data)
  .value("heif_suberror_Missing_grid_images", heif_suberror_Missing_grid_images)
  .value("heif_suberror_No_av1C_box", heif_suberror_No_av1C_box)
+ .value("heif_suberror_No_avcC_box", heif_suberror_No_avcC_box)
  .value("heif_suberror_Invalid_clean_aperture", heif_suberror_Invalid_clean_aperture)
  .value("heif_suberror_Invalid_overlay_data", heif_suberror_Invalid_overlay_data)
  .value("heif_suberror_Overlay_image_outside_of_canvas", heif_suberror_Overlay_image_outside_of_canvas)

libheif/codecs/avc.cc

@@ -23,6 +23,9 @@
 #include <iomanip>
 #include <iostream>
 #include <vector>
+#include "file.h"
+#include "context.h"
+
 
 Error Box_avcC::parse(BitstreamRange &range) {
  m_configuration.configuration_version = range.read8();
@@ -142,3 +145,153 @@ std::string Box_avcC::profileIndicationAsText() const {
  return "Unknown";
  }
 }
+
+
+void Box_avcC::get_header_nals(std::vector<uint8_t>& data) const
+{
+ for (const auto& sps : m_sps) {
+ data.push_back((sps.size() >> 24) & 0xFF);
+ data.push_back((sps.size() >> 16) & 0xFF);
+ data.push_back((sps.size() >> 8) & 0xFF);
+ data.push_back((sps.size() >> 0) & 0xFF);
+
+ data.insert(data.end(), sps.begin(), sps.end());
+ }
+
+ for (const auto& pps : m_pps) {
+ data.push_back((pps.size() >> 24) & 0xFF);
+ data.push_back((pps.size() >> 16) & 0xFF);
+ data.push_back((pps.size() >> 8) & 0xFF);
+ data.push_back((pps.size() >> 0) & 0xFF);
+
+ data.insert(data.end(), pps.begin(), pps.end());
+ }
+}
+
+
+
+Result<ImageItem::CodedImageData> ImageItem_AVC::encode(const std::shared_ptr<HeifPixelImage>& image,
+ struct heif_encoder* encoder,
+ const struct heif_encoding_options& options,
+ enum heif_image_input_class input_class)
+{
+#if 0
+ CodedImageData codedImage;
+
+ auto hvcC = std::make_shared<Box_hvcC>();
+
+ heif_image c_api_image;
+ c_api_image.image = image;
+
+ struct heif_error err = encoder->plugin->encode_image(encoder->encoder, &c_api_image, input_class);
+ if (err.code) {
+ return Error(err.code,
+ err.subcode,
+ err.message);
+ }
+
+ int encoded_width = 0;
+ int encoded_height = 0;
+
+ for (;;) {
+ uint8_t* data;
+ int size;
+
+ encoder->plugin->get_compressed_data(encoder->encoder, &data, &size, nullptr);
+
+ if (data == nullptr) {
+ break;
+ }
+
+
+ const uint8_t NAL_SPS = 33;
+
+ if ((data[0] >> 1) == NAL_SPS) {
+ Box_hvcC::configuration config;
+
+ parse_sps_for_hvcC_configuration(data, size, &config, &encoded_width, &encoded_height);
+
+ hvcC->set_configuration(config);
+
+ codedImage.encoded_image_width = encoded_width;
+ codedImage.encoded_image_height = encoded_height;
+ }
+
+ switch (data[0] >> 1) {
+ case 0x20:
+ case 0x21:
+ case 0x22:
+ hvcC->append_nal_data(data, size);
+ break;
+
+ default:
+ codedImage.append_with_4bytes_size(data, size);
+ // m_heif_file->append_iloc_data_with_4byte_size(image_id, data, size);
+ }
+ }
+
+ if (!encoded_width || !encoded_height) {
+ return Error(heif_error_Encoder_plugin_error,
+ heif_suberror_Invalid_image_size);
+ }
+
+ codedImage.properties.push_back(hvcC);
+
+
+ // Make sure that the encoder plugin works correctly and the encoded image has the correct size.
+
+ if (encoder->plugin->plugin_api_version >= 3 &&
+ encoder->plugin->query_encoded_size != nullptr) {
+ uint32_t check_encoded_width = image->get_width(), check_encoded_height = image->get_height();
+
+ encoder->plugin->query_encoded_size(encoder->encoder,
+ image->get_width(), image->get_height(),
+ &check_encoded_width,
+ &check_encoded_height);
+
+ assert((int)check_encoded_width == encoded_width);
+ assert((int)check_encoded_height == encoded_height);
+ }
+
+ return codedImage;
+#endif
+ assert(false); // TODO
+ return {};
+}
+
+Result<std::vector<uint8_t>> ImageItem_AVC::read_bitstream_configuration_data(heif_item_id itemId) const
+{
+ std::vector<uint8_t> data;
+
+ auto avcC_box = get_file()->get_property<Box_avcC>(itemId);
+ if (!avcC_box) {
+ return Error{heif_error_Invalid_input,
+ heif_suberror_No_av1C_box};
+ }
+
+ avcC_box->get_header_nals(data);
+
+ return data;
+}
+
+
+int ImageItem_AVC::get_luma_bits_per_pixel() const
+{
+ auto avcC_box = get_file()->get_property<Box_avcC>(get_id());
+ if (avcC_box) {
+ return 8; // TODO avcC_box->get_configuration().bit_depth_luma;
+ }
+
+ return -1;
+}
+
+
+int ImageItem_AVC::get_chroma_bits_per_pixel() const
+{
+ auto avcC_box = get_file()->get_property<Box_avcC>(get_id());
+ if (avcC_box) {
+ return 8; // TODO avcC_box->get_configuration().bit_depth_chroma;
+ }
+
+ return -1;
+}

libheif/codecs/avc.h

@@ -26,6 +26,8 @@
 #include <cstdint>
 #include <vector>
 #include <string>
+#include <memory>
+#include "codecs/image_item.h"
 
 class Box_avcC : public Box {
 public:
@@ -61,6 +63,7 @@ class Box_avcC : public Box {
  return m_pps;
  }
 
+ void get_header_nals(std::vector<uint8_t>& data) const;
 
  std::string dump(Indent &) const override;
 
@@ -77,4 +80,35 @@ class Box_avcC : public Box {
  std::vector< std::vector<uint8_t> > m_pps;
 };
 
+
+class ImageItem_AVC : public ImageItem
+{
+public:
+ ImageItem_AVC(HeifContext* ctx, heif_item_id id) : ImageItem(ctx, id) {}
+
+ ImageItem_AVC(HeifContext* ctx) : ImageItem(ctx) {}
+
+ const char* get_infe_type() const override { return "avc1"; }
+
+ const char* get_auxC_alpha_channel_type() const override { return "urn:mpeg:mpegB:cicp:systems:auxiliary:alpha"; }
+
+ const heif_color_profile_nclx* get_forced_output_nclx() const override { return nullptr; }
+
+ heif_compression_format get_compression_format() const override { return heif_compression_AVC; }
+
+ int get_luma_bits_per_pixel() const override;
+
+ int get_chroma_bits_per_pixel() const override;
+
+protected:
+ Result<std::vector<uint8_t>> read_bitstream_configuration_data(heif_item_id itemId) const override;
+
+public:
+
+ Result<CodedImageData> encode(const std::shared_ptr<HeifPixelImage>& image,
+ struct heif_encoder* encoder,
+ const struct heif_encoding_options& options,
+ enum heif_image_input_class input_class) override;
+};
+
 #endif

libheif/codecs/image_item.cc

@@ -25,6 +25,7 @@
 #include <codecs/jpeg.h>
 #include <codecs/jpeg2000.h>
 #include <codecs/avif.h>
+#include <codecs/avc.h>
 #include <codecs/hevc.h>
 #include <codecs/grid.h>
 #include <codecs/overlay.h>
@@ -121,6 +122,9 @@ std::shared_ptr<ImageItem> ImageItem::alloc_for_infe_box(HeifContext* ctx, const
  else if (item_type == "vvc1") {
  return std::make_shared<ImageItem_VVC>(ctx, id);
  }
+ else if (item_type == "avc1") {
+ return std::make_shared<ImageItem_AVC>(ctx, id);
+ }
 #if WITH_UNCOMPRESSED_CODEC
  else if (item_type == "unci") {
  return std::make_shared<ImageItem_uncompressed>(ctx, id);

libheif/context.cc

@@ -233,11 +233,12 @@ std::string HeifContext::debug_dump_boxes() const
 static bool item_type_is_image(const std::string& item_type, const std::string& content_type)
 {
  return (item_type == "hvc1" ||
+ item_type == "av01" ||
  item_type == "grid" ||
  item_type == "tild" ||
  item_type == "iden" ||
  item_type == "iovl" ||
- item_type == "av01" ||
+ item_type == "avc1" ||
  item_type == "unci" ||
  item_type == "vvc1" ||
  item_type == "jpeg" ||

libheif/error.cc

@@ -106,6 +106,8 @@ const char* Error::get_error_string(heif_suberror_code err)
  return "No 'vvcC' box";
  case heif_suberror_No_av1C_box:
  return "No 'av1C' box";
+ case heif_suberror_No_avcC_box:
+ return "No 'avcC' box";
  case heif_suberror_No_pitm_box:
  return "No 'pitm' box";
  case heif_suberror_No_ipco_box: