svcenc: Integer overflow in irc_ba_get_cur_frm_est_texture_bits

RC was incorrectly handling cases where 'I_TO_P_BIT_RATIO' multiplied by 'i4_est_texture_bits_for_frm' (which stores the estimated texture bits for the current frame), exceeded the range of int32_t. This has been fixed by clipping the product of the two quantities above. Bug = ossfuzz:63175 Test: svc_enc_fuzzer
ittiam-systems · Oct 16, 2023 · 41c0374 · 41c0374
1 parent 8121651
commit 41c0374
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/encoder/irc_bit_allocation.c b/encoder/irc_bit_allocation.c
@@ -35,6 +35,8 @@
 /** Macros **/
 #define MIN(x,y)  ((x) < (y))? (x) : (y)
 
+#define CLIP3(miny, maxy, y) (((y) < (miny)) ? (miny) : (((y) > (maxy)) ? (maxy) : (y)))
+
 /* State structure for bit allocation */
 typedef struct
 {
@@ -523,11 +525,11 @@ WORD32 irc_ba_get_cur_frm_est_texture_bits(bit_allocation_t *ps_bit_allocation,
         number_t_to_word32(vq_est_texture_bits_for_frm,
                            &i4_est_texture_bits_for_frm);
 
-        i4_est_texture_bits_for_frm =
-                        (I_PIC == e_pic_type) ?
-                                        (i4_est_texture_bits_for_frm
-                                                        * I_TO_P_BIT_RATIO) :
-                                        i4_est_texture_bits_for_frm;
+        i4_est_texture_bits_for_frm = (I_PIC == e_pic_type)
+                                          ? ((WORD32) CLIP3(INT32_MIN, INT32_MAX,
+                                                            ((WORD64) i4_est_texture_bits_for_frm) *
+                                                                ((WORD64) I_TO_P_BIT_RATIO)))
+                                          : i4_est_texture_bits_for_frm;
     }
 
     /*

diff --git a/encoder/irc_datatypes.h b/encoder/irc_datatypes.h
@@ -20,7 +20,7 @@
 
 #ifndef _RC_DATATYPES_H_
 #define _RC_DATATYPES_H_
-
+#include <stdint.h>
 
 /*****************************************************************************/
 /* Unsigned data types                                                       */
@@ -37,7 +37,7 @@ typedef unsigned long long   UWORD64;
 typedef signed char     WORD8;
 typedef short           WORD16;
 typedef int             WORD32;
-
+typedef int64_t WORD64;
 
 /*****************************************************************************/
 /* Miscellaneous data types                                                  */