fix overflows while encoding large images

The data type that was used to store the memory required for input raw
buffers was int. This works for normal dimensions. For images where
width and height are very large, say greater than 16000, then many
computations overflow causing issues.

This change also unifies the type for widths/heights/strides/buffer
sizes. Image dimensions and strides are represented using unsigned int
and buffer sizes are represented using size_t

Test: ./ultrahdr_unit_test

fuzzer/ultrahdr_dec_fuzzer.cpp

@@ -55,8 +55,8 @@ void UltraHdrDecFuzzer::process() {
   int bottom = mFdp.ConsumeIntegral<int16_t>();
 
   auto applyResize = mFdp.ConsumeBool();
-  int resizeWidth = mFdp.ConsumeIntegralInRange<int16_t>(-32, kMaxWidth + 128);
-  int resizeHeight = mFdp.ConsumeIntegralInRange<int16_t>(-32, kMaxHeight + 128);
+  int resizeWidth = mFdp.ConsumeIntegralInRange<int32_t>(-32, kMaxWidth + 128);
+  int resizeHeight = mFdp.ConsumeIntegralInRange<int32_t>(-32, kMaxHeight + 128);
 
   auto buffer = mFdp.ConsumeRemainingBytes<uint8_t>();
 

fuzzer/ultrahdr_enc_fuzzer.cpp

@@ -142,19 +142,19 @@ void UltraHdrEncFuzzer::process() {
 
     // raw buffer config
     bool hasHdrStride = mFdp.ConsumeBool();
-    int yHdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
+    size_t yHdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
     if (!hasHdrStride) yHdrStride = width;
     bool isHdrUVContiguous = mFdp.ConsumeBool();
     bool hasHdrUVStride = mFdp.ConsumeBool();
-    int uvHdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
+    size_t uvHdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
     if (!hasHdrUVStride) uvHdrStride = width;
 
     bool hasSdrStride = mFdp.ConsumeBool();
-    int ySdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
+    size_t ySdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
     if (!hasSdrStride) ySdrStride = width;
     bool isSdrUVContiguous = mFdp.ConsumeBool();
     bool hasSdrUVStride = mFdp.ConsumeBool();
-    int uvSdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width / 2, width / 2 + 128);
+    size_t uvSdrStride = mFdp.ConsumeIntegralInRange<uint16_t>(width / 2, width / 2 + 128);
     if (!hasSdrUVStride) uvSdrStride = width / 2;
 
     // editing effects
@@ -172,8 +172,8 @@ void UltraHdrEncFuzzer::process() {
     int bottom = mFdp.ConsumeIntegral<int16_t>();
 
     auto applyResize = mFdp.ConsumeBool();
-    int resizeWidth = mFdp.ConsumeIntegralInRange<int16_t>(-32, kMaxWidth + 128);
-    int resizeHeight = mFdp.ConsumeIntegralInRange<int16_t>(-32, kMaxHeight + 128);
+    int resizeWidth = mFdp.ConsumeIntegralInRange<int32_t>(-32, kMaxWidth + 128);
+    int resizeHeight = mFdp.ConsumeIntegralInRange<int32_t>(-32, kMaxHeight + 128);
 
     // exif
     char greeting[] = "Exif says hello world";

fuzzer/ultrahdr_legacy_fuzzer.cpp

@@ -134,19 +134,19 @@ void UltraHdrEncFuzzer::process() {
 
     // raw buffer config
     bool hasP010Stride = mFdp.ConsumeBool();
-    int yP010Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
+    size_t yP010Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
     if (!hasP010Stride) yP010Stride = width;
     bool isP010UVContiguous = mFdp.ConsumeBool();
     bool hasP010UVStride = mFdp.ConsumeBool();
-    int uvP010Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
+    size_t uvP010Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
     if (!hasP010UVStride) uvP010Stride = width;
 
     bool hasYuv420Stride = mFdp.ConsumeBool();
-    int yYuv420Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
+    size_t yYuv420Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width, width + 128);
     if (!hasYuv420Stride) yYuv420Stride = width;
     bool isYuv420UVContiguous = mFdp.ConsumeBool();
     bool hasYuv420UVStride = mFdp.ConsumeBool();
-    int uvYuv420Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width / 2, width / 2 + 128);
+    size_t uvYuv420Stride = mFdp.ConsumeIntegralInRange<uint16_t>(width / 2, width / 2 + 128);
     if (!hasYuv420UVStride) uvYuv420Stride = width / 2;
 
     // display boost
@@ -271,8 +271,8 @@ void UltraHdrEncFuzzer::process() {
                                reinterpret_cast<uint8_t*>(yuv420ImgCopy.chroma_data),
                                reinterpret_cast<uint8_t*>(yuv420ImgCopy.chroma_data) +
                                    yuv420ImgCopy.chroma_stride * yuv420ImgCopy.height / 2};
-      const size_t strides[3]{yuv420ImgCopy.luma_stride, yuv420ImgCopy.chroma_stride,
-                              yuv420ImgCopy.chroma_stride};
+      const unsigned int strides[3]{yuv420ImgCopy.luma_stride, yuv420ImgCopy.chroma_stride,
+                                    yuv420ImgCopy.chroma_stride};
       if (encoder
               .compressImage(planes, strides, yuv420ImgCopy.width, yuv420ImgCopy.height,
                              UHDR_IMG_FMT_12bppYCbCr420, quality, nullptr, 0)
@@ -291,7 +291,7 @@ void UltraHdrEncFuzzer::process() {
           jpegImgR.length = 0;
           JpegEncoderHelper gainMapEncoder;
           const uint8_t* planeGm[1]{reinterpret_cast<uint8_t*>(grayImg.data)};
-          const size_t strideGm[1]{grayImg.width};
+          const unsigned int strideGm[1]{grayImg.width};
           if (gainMapEncoder
                   .compressImage(planeGm, strideGm, grayImg.width, grayImg.height,
                                  UHDR_IMG_FMT_8bppYCbCr400, quality, nullptr, 0)

java/jni/ultrahdr-jni.cpp

@@ -388,6 +388,8 @@ Java_com_google_media_codecs_ultrahdr_UltraHDREncoder_getOutputNative(JNIEnv *en
                   "no output returned, may be call to uhdr_encode() was not made or encountered "
                   "error during encoding process.",
                   nullptr)
+  RET_VAL_IF_TRUE(enc_output->data_sz >= INT32_MAX, "java/lang/OutOfMemoryError",
+                  "encoded output size exceeds integer max", nullptr)
   jbyteArray output = env->NewByteArray(enc_output->data_sz);
   RET_VAL_IF_TRUE(output == nullptr, "java/io/IOException", "failed to allocate storage for output",
                   nullptr)

lib/include/ultrahdr/icc.h

@@ -180,7 +180,7 @@ static inline Fixed float_round_to_fixed(float x) {
 }
 
 static inline uint16_t float_round_to_unorm16(float x) {
-  x = x * 65535.f + 0.5;
+  x = x * 65535.f + 0.5f;
   if (x > 65535) return 65535;
   if (x < 0) return 0;
   return static_cast<uint16_t>(x);

lib/include/ultrahdr/jpegdecoderhelper.h

@@ -63,7 +63,7 @@ class JpegDecoderHelper {
    *
    * \return uhdr_error_info_t #UHDR_CODEC_OK if operation succeeds, uhdr_codec_err_t otherwise.
    */
-  uhdr_error_info_t decompressImage(const void* image, int length,
+  uhdr_error_info_t decompressImage(const void* image, size_t length,
                                     decode_mode_t mode = DECODE_TO_YCBCR_CS);
 
   /*!\brief This function parses the bitstream that is passed to it and makes image information
@@ -75,7 +75,7 @@ class JpegDecoderHelper {
    *
    * \return uhdr_error_info_t #UHDR_CODEC_OK if operation succeeds, uhdr_codec_err_t otherwise.
    */
-  uhdr_error_info_t parseImage(const void* image, int length) {
+  uhdr_error_info_t parseImage(const void* image, size_t length) {
     return decompressImage(image, length, PARSE_STREAM);
   }
 
@@ -99,13 +99,13 @@ class JpegDecoderHelper {
    * and it returned true. */
 
   /*!\brief returns image width */
-  size_t getDecompressedImageWidth() { return mPlaneWidth[0]; }
+  unsigned int getDecompressedImageWidth() { return mPlaneWidth[0]; }
 
   /*!\brief returns image height */
-  size_t getDecompressedImageHeight() { return mPlaneHeight[0]; }
+  unsigned int getDecompressedImageHeight() { return mPlaneHeight[0]; }
 
   /*!\brief returns number of components in image */
-  size_t getNumComponentsInImage() { return mNumComponents; }
+  unsigned int getNumComponentsInImage() { return mNumComponents; }
 
   /*!\brief returns pointer to xmp block present in input image */
   void* getXMPPtr() { return mXMPBuffer.data(); }
@@ -135,13 +135,13 @@ class JpegDecoderHelper {
    * via parseImage()/decompressImage() call. Note this does not include jpeg marker (0xffe1) and
    * the next 2 bytes indicating the size of the payload. If exif block is not present in the image
    * passed, then it returns -1. */
-  int getEXIFPos() { return mExifPayLoadOffset; }
+  long getEXIFPos() { return mExifPayLoadOffset; }
 
  private:
   // max number of components supported
   static constexpr int kMaxNumComponents = 3;
 
-  uhdr_error_info_t decode(const void* image, int length, decode_mode_t mode);
+  uhdr_error_info_t decode(const void* image, size_t length, decode_mode_t mode);
   uhdr_error_info_t decode(jpeg_decompress_struct* cinfo, uint8_t* dest);
   uhdr_error_info_t decodeToCSYCbCr(jpeg_decompress_struct* cinfo, uint8_t* dest);
   uhdr_error_info_t decodeToCSRGB(jpeg_decompress_struct* cinfo, uint8_t* dest);
@@ -157,14 +157,14 @@ class JpegDecoderHelper {
 
   // image attributes
   uhdr_img_fmt_t mOutFormat;
-  size_t mNumComponents;
-  size_t mPlaneWidth[kMaxNumComponents];
-  size_t mPlaneHeight[kMaxNumComponents];
-  size_t mPlaneHStride[kMaxNumComponents];
-  size_t mPlaneVStride[kMaxNumComponents];
-
-  int mExifPayLoadOffset;  // Position of EXIF package, default value is -1 which means no EXIF
-                           // package appears.
+  unsigned int mNumComponents;
+  unsigned int mPlaneWidth[kMaxNumComponents];
+  unsigned int mPlaneHeight[kMaxNumComponents];
+  unsigned int mPlaneHStride[kMaxNumComponents];
+  unsigned int mPlaneVStride[kMaxNumComponents];
+
+  long mExifPayLoadOffset;  // Position of EXIF package, default value is -1 which means no EXIF
+                            // package appears.
 };
 
 } /* namespace ultrahdr  */

lib/include/ultrahdr/jpegencoderhelper.h

@@ -61,7 +61,7 @@ class JpegEncoderHelper {
    * \return uhdr_error_info_t #UHDR_CODEC_OK if operation succeeds, uhdr_codec_err_t otherwise.
    */
   uhdr_error_info_t compressImage(const uhdr_raw_image_t* img, const int qfactor,
-                                  const void* iccBuffer, const unsigned int iccSize);
+                                  const void* iccBuffer, const size_t iccSize);
 
   /*!\brief This function encodes the raw image that is passed to it and stores the results
    * internally. The result is accessible via getter functions.
@@ -77,10 +77,9 @@ class JpegEncoderHelper {
    *
    * \return uhdr_error_info_t #UHDR_CODEC_OK if operation succeeds, uhdr_codec_err_t otherwise.
    */
-  uhdr_error_info_t compressImage(const uint8_t* planes[3], const size_t strides[3],
+  uhdr_error_info_t compressImage(const uint8_t* planes[3], const unsigned int strides[3],
                                   const int width, const int height, const uhdr_img_fmt_t format,
-                                  const int qfactor, const void* iccBuffer,
-                                  const unsigned int iccSize);
+                                  const int qfactor, const void* iccBuffer, const size_t iccSize);
 
   /*! Below public methods are only effective if a call to compressImage() is made and it returned
    * true. */
@@ -102,20 +101,20 @@ class JpegEncoderHelper {
   // max number of components supported
   static constexpr int kMaxNumComponents = 3;
 
-  uhdr_error_info_t encode(const uint8_t* planes[3], const size_t strides[3], const int width,
+  uhdr_error_info_t encode(const uint8_t* planes[3], const unsigned int strides[3], const int width,
                            const int height, const uhdr_img_fmt_t format, const int qfactor,
-                           const void* iccBuffer, const unsigned int iccSize);
+                           const void* iccBuffer, const size_t iccSize);
 
   uhdr_error_info_t compressYCbCr(jpeg_compress_struct* cinfo, const uint8_t* planes[3],
-                                  const size_t strides[3]);
+                                  const unsigned int strides[3]);
 
   destination_mgr_impl mDestMgr;  // object for managing output
 
   // temporary storage
   std::unique_ptr<uint8_t[]> mPlanesMCURow[kMaxNumComponents];
 
-  size_t mPlaneWidth[kMaxNumComponents];
-  size_t mPlaneHeight[kMaxNumComponents];
+  unsigned int mPlaneWidth[kMaxNumComponents];
+  unsigned int mPlaneHeight[kMaxNumComponents];
 };
 
 } /* namespace ultrahdr  */

lib/include/ultrahdr/jpegr.h

@@ -30,8 +30,8 @@ namespace ultrahdr {
 
 // Default configurations
 // gainmap image downscale factor
-static const size_t kMapDimensionScaleFactorDefault = 1;
-static const size_t kMapDimensionScaleFactorAndroidDefault = 4;
+static const int kMapDimensionScaleFactorDefault = 1;
+static const int kMapDimensionScaleFactorAndroidDefault = 4;
 
 // JPEG compress quality (0 ~ 100) for base image
 static const int kBaseCompressQualityDefault = 95;
@@ -63,17 +63,17 @@ struct jpeg_info_struct {
   std::vector<uint8_t> exifData = std::vector<uint8_t>(0);
   std::vector<uint8_t> xmpData = std::vector<uint8_t>(0);
   std::vector<uint8_t> isoData = std::vector<uint8_t>(0);
-  size_t width;
-  size_t height;
-  size_t numComponents;
+  unsigned int width;
+  unsigned int height;
+  unsigned int numComponents;
 };
 
 /*
  * Holds information of jpegr image
  */
 struct jpegr_info_struct {
-  size_t width;   // copy of primary image width (for easier access)
-  size_t height;  // copy of primary image height (for easier access)
+  unsigned int width;   // copy of primary image width (for easier access)
+  unsigned int height;  // copy of primary image height (for easier access)
   jpeg_info_struct* primaryImgInfo = nullptr;
   jpeg_info_struct* gainmapImgInfo = nullptr;
 };
@@ -84,7 +84,7 @@ typedef struct jpegr_info_struct* jr_info_ptr;
 class JpegR {
  public:
   JpegR(void* uhdrGLESCtxt = nullptr,
-        size_t mapDimensionScaleFactor = kMapDimensionScaleFactorAndroidDefault,
+        int mapDimensionScaleFactor = kMapDimensionScaleFactorAndroidDefault,
         int mapCompressQuality = kMapCompressQualityAndroidDefault,
         bool useMultiChannelGainMap = kUseMultiChannelGainMapAndroidDefault,
         float gamma = kGainMapGammaDefault,
@@ -260,7 +260,7 @@ class JpegR {
    *
    * \return none
    */
-  void setMapDimensionScaleFactor(size_t mapDimensionScaleFactor) {
+  void setMapDimensionScaleFactor(int mapDimensionScaleFactor) {
     this->mMapDimensionScaleFactor = mapDimensionScaleFactor;
   }
 
@@ -269,7 +269,7 @@ class JpegR {
    *
    * \return mapDimensionScaleFactor
    */
-  size_t getMapDimensionScaleFactor() { return this->mMapDimensionScaleFactor; }
+  int getMapDimensionScaleFactor() { return this->mMapDimensionScaleFactor; }
 
   /*!\brief set gain map compression quality factor
    * NOTE: Applicable only in encoding scenario
@@ -415,8 +415,9 @@ class JpegR {
    *
    * \return uhdr_error_info_t #UHDR_CODEC_OK if operation succeeds, uhdr_codec_err_t otherwise.
    */
-  uhdr_error_info_t parseGainMapMetadata(uint8_t* iso_data, int iso_size, uint8_t* xmp_data,
-                                         int xmp_size, uhdr_gainmap_metadata_ext_t* uhdr_metadata);
+  uhdr_error_info_t parseGainMapMetadata(uint8_t* iso_data, size_t iso_size, uint8_t* xmp_data,
+                                         size_t xmp_size,
+                                         uhdr_gainmap_metadata_ext_t* uhdr_metadata);
 
   /*!\brief This method is used to tone map a hdr image
    *
@@ -514,7 +515,8 @@ class JpegR {
    * \return uhdr_error_info_t #UHDR_CODEC_OK if operation succeeds, uhdr_codec_err_t otherwise.
    */
   uhdr_error_info_t parseJpegInfo(uhdr_compressed_image_t* jpeg_image, j_info_ptr image_info,
-                                  size_t* img_width = nullptr, size_t* img_height = nullptr);
+                                  unsigned int* img_width = nullptr,
+                                  unsigned int* img_height = nullptr);
 
   /*!\brief This method takes compressed sdr intent, compressed gainmap coefficient, gainmap
    * metadata and creates a ultrahdr image. This is done by first generating XMP packet from gainmap
@@ -597,7 +599,7 @@ class JpegR {
 
   // Configurations
   void* mUhdrGLESCtxt;              // opengl es context
-  size_t mMapDimensionScaleFactor;  // gain map scale factor
+  int mMapDimensionScaleFactor;     // gain map scale factor
   int mMapCompressQuality;          // gain map quality factor
   bool mUseMultiChannelGainMap;     // enable multichannel gain map
   float mGamma;                     // gain map gamma parameter

lib/include/ultrahdr/jpegrutils.h

@@ -39,20 +39,20 @@ static inline uint16_t EndianSwap16(uint16_t value) {
 class DataStruct {
  private:
   void* data;
-  int writePos;
-  int length;
+  size_t writePos;
+  size_t length;
 
  public:
   DataStruct(int s);
   ~DataStruct();
 
   void* getData();
-  int getLength();
-  int getBytesWritten();
+  size_t getLength();
+  size_t getBytesWritten();
   bool write8(uint8_t value);
   bool write16(uint16_t value);
   bool write32(uint32_t value);
-  bool write(const void* src, int size);
+  bool write(const void* src, size_t size);
 };
 
 /*
@@ -64,8 +64,8 @@ class DataStruct {
  * @param position cursor in desitination where the data is to be written.
  * @return success or error code.
  */
-uhdr_error_info_t Write(uhdr_compressed_image_t* destination, const void* source, int length,
-                        int& position);
+uhdr_error_info_t Write(uhdr_compressed_image_t* destination, const void* source, size_t length,
+                        size_t& position);
 
 /*
  * Parses XMP packet and fills metadata with data from XMP
@@ -75,7 +75,7 @@ uhdr_error_info_t Write(uhdr_compressed_image_t* destination, const void* source
  * @param metadata place to store HDR metadata values
  * @return success or error code.
  */
-uhdr_error_info_t getMetadataFromXMP(uint8_t* xmp_data, int xmp_size,
+uhdr_error_info_t getMetadataFromXMP(uint8_t* xmp_data, size_t xmp_size,
                                      uhdr_gainmap_metadata_ext_t* metadata);
 
 /*
@@ -118,7 +118,7 @@ uhdr_error_info_t getMetadataFromXMP(uint8_t* xmp_data, int xmp_size,
  * @param secondary_image_length length of secondary image
  * @return XMP metadata in type of string
  */
-std::string generateXmpForPrimaryImage(int secondary_image_length,
+std::string generateXmpForPrimaryImage(size_t secondary_image_length,
                                        uhdr_gainmap_metadata_ext_t& metadata);
 
 /*

lib/include/ultrahdr/multipictureformat.h

@@ -69,8 +69,8 @@ constexpr uint32_t kMPEntryAttributeFormatJpeg = 0x0000000;
 constexpr uint32_t kMPEntryAttributeTypePrimary = 0x030000;
 
 size_t calculateMpfSize();
-std::shared_ptr<DataStruct> generateMpf(int primary_image_size, int primary_image_offset,
-                                        int secondary_image_size, int secondary_image_offset);
+std::shared_ptr<DataStruct> generateMpf(size_t primary_image_size, size_t primary_image_offset,
+                                        size_t secondary_image_size, size_t secondary_image_offset);
 
 }  // namespace ultrahdr