Skip to content

Commit

Permalink
use matrix coeffs from icc tag while decoding
Browse files Browse the repository at this point in the history 
icc tag does not signal matrix coefficients directly. They signal color
primaries. From these color primaries, one can derive the matrix
coefficients needed for yuv to rgb conversion using, equations 39-44 of
itu h273. In this process bradford chroma adaptation matrix also plays a
role. The current change does not do all this. Instead it does a look up
mapping of icc tag to color gamut. Successfull harvesting of matrix
coefficients from icc tags is dependent on the color gamut dictionary
maintained internally. For now, this is okish. If gamut map fails, we
will default to srgb color space as per iso 21496-1 sec c.4.4

Test: ./ultrahdr_unit_test

Co-authored-by: Vivek R Jadhav <[email protected]>
Change-Id: I41cb5013c5d29d5ebaa535a78e45a9b487a3c122
  • Loading branch information
@ram-mohan @vivekrj1806
ram-mohan and vivekrj1806 committed Dec 10, 2024
1 parent fc6b310 commit 0d64f2b
Show file tree
Hide file tree
Showing 5 changed files with 183 additions and 36 deletions.
2 changes: 2 additions & 0 deletions lib/include/ultrahdr/gainmapmath.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,8 @@ std::unique_ptr<uhdr_raw_image_ext_t> convert_raw_input_to_ycbcr(
std::unique_ptr<uhdr_raw_image_ext_t> convert_raw_input_to_ycbcr_neon(uhdr_raw_image_t* src);
#endif

uhdr_error_info_t convert_ycbcr_input_to_rgb(uhdr_raw_image_t* src, uhdr_raw_image_t* dst);

bool floatToSignedFraction(float v, int32_t* numerator, uint32_t* denominator);
bool floatToUnsignedFraction(float v, uint32_t* numerator, uint32_t* denominator);

Expand Down
66 changes: 66 additions & 0 deletions lib/src/gainmapmath.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1578,6 +1578,72 @@ std::unique_ptr<uhdr_raw_image_ext_t> convert_raw_input_to_ycbcr(uhdr_raw_image_
return dst;
}

uhdr_error_info_t convert_ycbcr_input_to_rgb(uhdr_raw_image_t* src, uhdr_raw_image_t* dst) {
if (dst->w != src->w || dst->h != src->h) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_MEM_ERROR;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"destination image dimensions %dx%d and source image dimensions %dx%d are not "
"identical for copy_raw_image",
dst->w, dst->h, src->w, src->h);
return status;
}

if (isPixelFormatRgb(src->fmt) || !isPixelFormatRgb(dst->fmt)) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"Unexpected source or destination color format, src fmt %d, dst fmt %d", src->fmt,
dst->fmt);
return status;
}

GetPixelFn get_pixel_fn = getPixelFn(src->fmt);
if (get_pixel_fn == nullptr) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for reading pixels for color format %d", src->fmt);
return status;
}

PutPixelFn put_pixel_fn = putPixelFn(dst->fmt);
if (put_pixel_fn == nullptr) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_UNSUPPORTED_FEATURE;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"No implementation available for writing pixels for color format %d", dst->fmt);
return status;
}

Color (*yuvToRgb)(Color) = nullptr;
if (src->cg == UHDR_CG_BT_709) {
yuvToRgb = srgbYuvToRgb;
} else if (src->cg == UHDR_CG_BT_2100) {
yuvToRgb = bt2100YuvToRgb;
} else if (src->cg == UHDR_CG_DISPLAY_P3) {
yuvToRgb = p3YuvToRgb;
} else {
yuvToRgb = Bt601YuvToRgb;
}

dst->cg = src->cg;
dst->ct = src->ct;
dst->range = UHDR_CR_FULL_RANGE;
for (unsigned int i = 0; i < src->h; i++) {
for (unsigned int j = 0; j < src->w; j++) {
Color yuv = get_pixel_fn(src, j, i);
Color rgb = yuvToRgb(yuv);
put_pixel_fn(dst, j, i, rgb);
}
}
return g_no_error;
}

std::unique_ptr<uhdr_raw_image_ext_t> copy_raw_image(uhdr_raw_image_t* src) {
std::unique_ptr<uhdr_raw_image_ext_t> dst = std::make_unique<ultrahdr::uhdr_raw_image_ext_t>(
src->fmt, src->cg, src->ct, src->range, src->w, src->h, 64);
Expand Down
46 changes: 33 additions & 13 deletions lib/src/gpu/applygainmap_gl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,6 @@ static const std::string getYuv420PixelShader = R"__SHADER__(
}
)__SHADER__";

static const std::string p3YUVToRGBShader = R"__SHADER__(
vec3 p3YuvToRgb(const vec3 color) {
const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);
const mat3 transform = mat3(
1.0, 1.0, 1.0,
0.0, -0.344136286, 1.772,
1.402, -0.714136286, 0.0);
return clamp(transform * (color - offset), 0.0, 1.0);
}
)__SHADER__";

static const std::string sRGBEOTFShader = R"__SHADER__(
float sRGBEOTF(float e_gamma) {
return e_gamma <= 0.04045 ? e_gamma / 12.92 : pow((e_gamma + 0.055) / 1.055, 2.4);
Expand Down Expand Up @@ -191,6 +180,13 @@ static const std::string hlgInverseOOTFShader = R"__SHADER__(
}
)__SHADER__";

const std::array<float, 9> kBt709YUVToRGB = {1.0f, 0.0f, 1.5748f, 1.0f, -0.18732f,
-0.46812f, 1.0f, 1.8556f, 0.0f};
const std::array<float, 9> kP3YUVToRGB = {1.0f, 0.0f, 1.542051f, 1.0f, -0.21106f,
-0.51044f, 1.0f, 1.841426f, 0.0f};
const std::array<float, 9> kBt2100YUVToRGB = {1.0f, 0.0f, 1.4746f, 1.0f, -0.16455f,
-0.57135f, 1.0f, 1.8814f, 0.0f};

template <typename... Args>
std::string StringFormat(const std::string& format, Args... args) {
auto size = std::snprintf(nullptr, 0, format.c_str(), args...);
Expand All @@ -200,6 +196,30 @@ std::string StringFormat(const std::string& format, Args... args) {
return std::string(buffer.data(), size); // Exclude the terminating null byte
}

std::string getYuvToRgbShader(uhdr_color_gamut_t cg) {
const float* coeffs = nullptr;
if (cg == UHDR_CG_BT_709) {
coeffs = kBt709YUVToRGB.data();
} else if (cg == UHDR_CG_DISPLAY_P3) {
coeffs = kP3YUVToRGB.data();
} else if (cg == UHDR_CG_BT_2100) {
coeffs = kBt2100YUVToRGB.data();
} else {
coeffs = kBt709YUVToRGB.data();
}
return StringFormat(
" vec3 sdrYuvToRgb(const vec3 color) {\n"
" const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);\n"
" const mat3 transform = mat3(\n"
" %f, %f, %f,\n"
" %f, %f, %f,\n"
" %f, %f, %f);\n"
" return clamp(transform * (color - offset), 0.0, 1.0);\n"
" }\n",
coeffs[0], coeffs[3], coeffs[6], coeffs[1], coeffs[4], coeffs[7], coeffs[2], coeffs[5],
coeffs[8]);
}

std::string getClampPixelFloatShader(uhdr_color_transfer_t output_ct) {
return StringFormat(
" vec3 clampPixelFloat(const vec3 color) {\n"
Expand Down Expand Up @@ -260,7 +280,7 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_
} else if (sdr_fmt == UHDR_IMG_FMT_12bppYCbCr420) {
shader_code.append(getYuv420PixelShader);
}
shader_code.append(p3YUVToRGBShader);
shader_code.append(getYuvToRgbShader(sdr_cg));
shader_code.append(sRGBEOTFShader);
shader_code.append(gm_fmt == UHDR_IMG_FMT_8bppYCbCr400 ? getGainMapSampleSingleChannel
: getGainMapSampleMultiChannel);
Expand All @@ -276,7 +296,7 @@ std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_
shader_code.append(R"__SHADER__(
void main() {
vec3 yuv_gamma_sdr = getYUVPixel();
vec3 rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
vec3 rgb_gamma_sdr = sdrYuvToRgb(yuv_gamma_sdr);
vec3 rgb_sdr = sRGBEOTF(rgb_gamma_sdr);
)__SHADER__");
if (sdr_cg != hdr_cg && !use_base_cg) {
Expand Down
97 changes: 74 additions & 23 deletions lib/src/jpegr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,13 +271,13 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent, uhdr_raw_imag
}

// convert to bt601 YUV encoding for JPEG encode
#if (defined(UHDR_ENABLE_INTRINSICS) && (defined(__ARM_NEON__) || defined(__ARM_NEON)))
UHDR_ERR_CHECK(
convertYuv_neon(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
#else
UHDR_ERR_CHECK(
convertYuv(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
#endif
//#if (defined(UHDR_ENABLE_INTRINSICS) && (defined(__ARM_NEON__) || defined(__ARM_NEON)))
// UHDR_ERR_CHECK(
// convertYuv_neon(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
//#else
// UHDR_ERR_CHECK(
// convertYuv(sdr_intent_yuv, sdr_intent_yuv->cg, (uhdr_color_gamut_t)UHDR_CG_BT_601));
//#endif

// compress sdr image
JpegEncoderHelper jpeg_enc_obj_sdr;
Expand Down Expand Up @@ -312,6 +312,33 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent, uhdr_raw_imag
return status;
}

if (jpeg_dec_obj_sdr.getICCSize() > 0) {
uhdr_color_gamut_t cg =
IccHelper::readIccColorGamut(jpeg_dec_obj_sdr.getICCPtr(), jpeg_dec_obj_sdr.getICCSize());
if (cg == UHDR_CG_UNSPECIFIED ||
(sdr_intent_compressed->cg != UHDR_CG_UNSPECIFIED && sdr_intent_compressed->cg != cg) ||
sdr_intent->cg != cg) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_INVALID_PARAM;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail,
"configured color gamut %d does not match with color gamut specified in icc box %d",
sdr_intent->cg, cg);
return status;
}
} else {
if (sdr_intent_compressed->cg != UHDR_CG_UNSPECIFIED &&
sdr_intent_compressed->cg != sdr_intent->cg) {
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_INVALID_PARAM;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail, "Unrecognized sdr intent color gamut %d",
sdr_intent_compressed->cg);
return status;
}
}
sdr_intent_compressed->cg = sdr_intent->cg;

// generate gain map
uhdr_gainmap_metadata_ext_t metadata(kJpegrVersion);
std::unique_ptr<uhdr_raw_image_ext_t> gainmap;
Expand All @@ -322,17 +349,27 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent, uhdr_raw_imag
UHDR_ERR_CHECK(compressGainMap(gainmap.get(), &jpeg_enc_obj_gm));
uhdr_compressed_image_t gainmap_compressed = jpeg_enc_obj_gm.getCompressedImage();

return encodeJPEGR(sdr_intent_compressed, &gainmap_compressed, &metadata, dest);
// Add ICC if not already present.
if (jpeg_dec_obj_sdr.getICCSize() > 0) {
UHDR_ERR_CHECK(appendGainMap(sdr_intent_compressed, &gainmap_compressed, /* exif */ nullptr,
/* icc */ nullptr, /* icc size */ 0, &metadata, dest));
} else {
std::shared_ptr<DataStruct> newIcc =
IccHelper::writeIccProfile(UHDR_CT_SRGB, sdr_intent_compressed->cg);
UHDR_ERR_CHECK(appendGainMap(sdr_intent_compressed, &gainmap_compressed, /* exif */ nullptr,
newIcc->getData(), newIcc->getLength(), &metadata, dest));
}

return g_no_error;
}

/* Encode API-3 */
uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent,
uhdr_compressed_image_t* sdr_intent_compressed,
uhdr_compressed_image_t* dest) {
// decode input jpeg, gamut is going to be bt601.
JpegDecoderHelper jpeg_dec_obj_sdr;
UHDR_ERR_CHECK(jpeg_dec_obj_sdr.decompressImage(sdr_intent_compressed->data,
sdr_intent_compressed->data_sz));
UHDR_ERR_CHECK(jpeg_dec_obj_sdr.decompressImage(
sdr_intent_compressed->data, sdr_intent_compressed->data_sz, DECODE_TO_YCBCR_CS));

uhdr_raw_image_t sdr_intent = jpeg_dec_obj_sdr.getDecompressedImage();
if (jpeg_dec_obj_sdr.getICCSize() > 0) {
Expand All @@ -355,7 +392,7 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent,
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_INVALID_PARAM;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail, "Unrecognized 420 color gamut %d",
snprintf(status.detail, sizeof status.detail, "Unrecognized sdr intent color gamut %d",
sdr_intent_compressed->cg);
return status;
}
Expand All @@ -375,15 +412,25 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_raw_image_t* hdr_intent,
// generate gain map
uhdr_gainmap_metadata_ext_t metadata(kJpegrVersion);
std::unique_ptr<uhdr_raw_image_ext_t> gainmap;
UHDR_ERR_CHECK(
generateGainMap(&sdr_intent, hdr_intent, &metadata, gainmap, true /* sdr_is_601 */));
UHDR_ERR_CHECK(generateGainMap(&sdr_intent, hdr_intent, &metadata, gainmap));

// compress gain map
JpegEncoderHelper jpeg_enc_obj_gm;
UHDR_ERR_CHECK(compressGainMap(gainmap.get(), &jpeg_enc_obj_gm));
uhdr_compressed_image_t gainmap_compressed = jpeg_enc_obj_gm.getCompressedImage();

return encodeJPEGR(sdr_intent_compressed, &gainmap_compressed, &metadata, dest);
// Add ICC if not already present.
if (jpeg_dec_obj_sdr.getICCSize() > 0) {
UHDR_ERR_CHECK(appendGainMap(sdr_intent_compressed, &gainmap_compressed, /* exif */ nullptr,
/* icc */ nullptr, /* icc size */ 0, &metadata, dest));
} else {
std::shared_ptr<DataStruct> newIcc =
IccHelper::writeIccProfile(UHDR_CT_SRGB, sdr_intent_compressed->cg);
UHDR_ERR_CHECK(appendGainMap(sdr_intent_compressed, &gainmap_compressed, /* exif */ nullptr,
newIcc->getData(), newIcc->getLength(), &metadata, dest));
}

return g_no_error;
}

/* Encode API-4 */
Expand Down Expand Up @@ -422,7 +469,7 @@ uhdr_error_info_t JpegR::encodeJPEGR(uhdr_compressed_image_t* base_img_compresse
uhdr_error_info_t status;
status.error_code = UHDR_CODEC_INVALID_PARAM;
status.has_detail = 1;
snprintf(status.detail, sizeof status.detail, "Unrecognized 420 color gamut %d",
snprintf(status.detail, sizeof status.detail, "Unrecognized sdr intent color gamut %d",
base_img_compressed->cg);
return status;
}
Expand Down Expand Up @@ -1418,9 +1465,8 @@ uhdr_error_info_t JpegR::decodeJPEGR(uhdr_compressed_image_t* uhdr_compressed_im
extractPrimaryImageAndGainMap(uhdr_compressed_img, &primary_jpeg_image, &gainmap_jpeg_image))

JpegDecoderHelper jpeg_dec_obj_sdr;
UHDR_ERR_CHECK(jpeg_dec_obj_sdr.decompressImage(
primary_jpeg_image.data, primary_jpeg_image.data_sz,
(output_ct == UHDR_CT_SRGB) ? DECODE_TO_RGB_CS : DECODE_TO_YCBCR_CS));
UHDR_ERR_CHECK(jpeg_dec_obj_sdr.decompressImage(primary_jpeg_image.data,
primary_jpeg_image.data_sz, DECODE_TO_YCBCR_CS));

JpegDecoderHelper jpeg_dec_obj_gm;
uhdr_raw_image_t gainmap;
Expand Down Expand Up @@ -1461,7 +1507,7 @@ uhdr_error_info_t JpegR::decodeJPEGR(uhdr_compressed_image_t* uhdr_compressed_im
sdr_intent.cg =
IccHelper::readIccColorGamut(jpeg_dec_obj_sdr.getICCPtr(), jpeg_dec_obj_sdr.getICCSize());
if (output_ct == UHDR_CT_SRGB) {
UHDR_ERR_CHECK(copy_raw_image(&sdr_intent, dest));
UHDR_ERR_CHECK(convert_ycbcr_input_to_rgb(&sdr_intent, dest));
return g_no_error;
}

Expand Down Expand Up @@ -1599,23 +1645,28 @@ uhdr_error_info_t JpegR::applyGainMap(uhdr_raw_image_t* sdr_intent, uhdr_raw_ima
return status;
}

ColorTransformFn sdrYuvToRgbFn = getYuvToRgbFn(sdr_intent->cg);
// If failed to read color space, default to srgb color space as per iso 21496-1 sec C.4.4
if (sdrYuvToRgbFn == nullptr) {
sdrYuvToRgbFn = srgbYuvToRgb;
}

JobQueue jobQueue;
std::function<void()> applyRecMap = [sdr_intent, gainmap_img, dest, &jobQueue, &idwTable,
output_ct, &gainLUT, gainmap_metadata, hdrGamutConversionFn,
sdrGamutConversionFn,
#if !USE_APPLY_GAIN_LUT
gainmap_weight,
#endif
map_scale_factor, get_pixel_fn]() -> void {
map_scale_factor, get_pixel_fn, sdrYuvToRgbFn]() -> void {
unsigned int width = sdr_intent->w;
unsigned int rowStart, rowEnd;

while (jobQueue.dequeueJob(rowStart, rowEnd)) {
for (size_t y = rowStart; y < rowEnd; ++y) {
for (size_t x = 0; x < width; ++x) {
Color yuv_gamma_sdr = get_pixel_fn(sdr_intent, x, y);
// Assuming the sdr image is a decoded JPEG, we should always use Rec.601 YUV coefficients
Color rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
Color rgb_gamma_sdr = sdrYuvToRgbFn(yuv_gamma_sdr);
// We are assuming the SDR base image is always sRGB transfer.
#if USE_SRGB_INVOETF_LUT
Color rgb_sdr = srgbInvOetfLUT(rgb_gamma_sdr);
Expand Down
8 changes: 8 additions & 0 deletions tests/jpegr_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ class UhdrCompressedStructWrapper {
~UhdrCompressedStructWrapper() = default;

bool allocateMemory();
bool setImageColorGamut(ultrahdr_color_gamut colorGamut);
jr_compressed_ptr getImageHandle();

private:
Expand Down Expand Up @@ -275,6 +276,11 @@ bool UhdrCompressedStructWrapper::allocateMemory() {
return true;
}

bool UhdrCompressedStructWrapper::setImageColorGamut(ultrahdr_color_gamut colorGamut) {
mImg.colorGamut = colorGamut;
return true;
}

jr_compressed_ptr UhdrCompressedStructWrapper::getImageHandle() { return &mImg; }

#ifdef DUMP_OUTPUT
Expand Down Expand Up @@ -1854,6 +1860,7 @@ TEST_P(JpegRAPIEncodeAndDecodeTest, EncodeAPI2AndDecodeTest) {
ASSERT_TRUE(jpgImg.allocateMemory());
UhdrCompressedStructWrapper jpgSdr(kImageWidth, kImageHeight);
ASSERT_TRUE(jpgSdr.allocateMemory());
ASSERT_TRUE(jpgSdr.setImageColorGamut(mYuv420ColorGamut));
auto sdr = jpgSdr.getImageHandle();
ASSERT_TRUE(readFile(kSdrJpgFileName, sdr->data, sdr->maxLength, sdr->length));
JpegR uHdrLib;
Expand Down Expand Up @@ -2056,6 +2063,7 @@ TEST_P(JpegRAPIEncodeAndDecodeTest, EncodeAPI3AndDecodeTest) {
ASSERT_TRUE(jpgImg.allocateMemory());
UhdrCompressedStructWrapper jpgSdr(kImageWidth, kImageHeight);
ASSERT_TRUE(jpgSdr.allocateMemory());
ASSERT_TRUE(jpgSdr.setImageColorGamut(mYuv420ColorGamut));
auto sdr = jpgSdr.getImageHandle();
ASSERT_TRUE(readFile(kSdrJpgFileName, sdr->data, sdr->maxLength, sdr->length));
JpegR uHdrLib;
Expand Down

0 comments on commit 0d64f2b

Please sign in to comment.