UBSAN: shift-out-of-bounds in rtl8188eus/hal/phydm/phydm_phystatus.c when loading rtl8818eu wifi usb adaptor

[sardChen]

When executing fuzzing test for loading rtl8818eu, I found two shift-out-of-bounds bugs reported by UBSAN in dmesg logs:

[  334.038673] ================================================================================
[  334.038711] UBSAN: shift-out-of-bounds in /home/sandy/workplace/wifiTool/driver/rtl8188eus/hal/phydm/phydm_phystatus.c:1751:67
[  334.038729] shift exponent 63 is too large for 32-bit type 'int'
[  334.038743] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G           OE      6.6.58 #1
[  334.038756] Hardware name: Gigabyte Technology Co., Ltd. B660M GAMING AC DDR4/B660M GAMING AC DDR4, BIOS F4 01/17/2022
[  334.038763] Call Trace:
[  334.038769]  <IRQ>
[  334.038777]  dump_stack_lvl+0x48/0x70
[  334.038801]  dump_stack+0x10/0x20
[  334.038814]  __ubsan_handle_shift_out_of_bounds+0x156/0x310
[  334.038828]  ? ethnl_set_plca+0x756/0x9d0
[  334.038847]  phydm_process_rssi_for_dm+0x6b0/0xb00 [8188eu]
[  334.039198]  odm_phy_status_query+0x2bc/0x550 [8188eu]
[  334.039518]  rx_query_phy_status+0x4e5/0x990 [8188eu]
[  334.039893]  ? __pfx_rx_query_phy_status+0x10/0x10 [8188eu]
[  334.040245]  ? __asan_memcpy+0x4e/0x80
[  334.040258]  ? _rtw_memcpy+0x10/0x20 [8188eu]
[  334.040657]  pre_recv_entry+0x77/0x150 [8188eu]
[  334.041005]  recvbuf2recvframe+0x5b2/0x710 [8188eu]
[  334.041368]  usb_recv_tasklet+0x12b/0x230 [8188eu]
[  334.041776]  tasklet_action_common.constprop.0+0x275/0x670
[  334.041795]  tasklet_action+0x22/0x30
[  334.041808]  handle_softirqs+0x192/0x5d0
[  334.041825]  __irq_exit_rcu+0x15c/0x1b0
[  334.041838]  irq_exit_rcu+0xe/0x20
[  334.041850]  common_interrupt+0xa4/0xb0
[  334.041861]  </IRQ>
[  334.041866]  <TASK>
[  334.041873]  asm_common_interrupt+0x27/0x40
[  334.041883] RIP: 0010:cpuidle_enter_state+0x1df/0x520
[  334.041895] Code: 01 73 dd 48 83 c4 28 44 89 f0 5b 41 5c 41 5d 41 5e 41 5f 5d 31 d2 31 c9 31 f6 31 ff 45 31 c0 c3 cc cc cc cc fb 0f 1f 44 00 00 <45> 85 f6 0f 89 06 ff ff ff 48 c7 43 18 00 00 00 00 49 83 fd 09 0f
[  334.041905] RSP: 0018:ffff888100dd7d30 EFLAGS: 00000246
[  334.041919] RAX: 0000000000000000 RBX: ffff88885c34ffe0 RCX: 0000000000000000
[  334.041928] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[  334.041935] RBP: ffff888100dd7d80 R08: 0000000000000000 R09: 0000000000000000
[  334.041941] R10: 0000000000000000 R11: 0000000000000000 R12: ffffffffb50627c0
[  334.041948] R13: 0000000000000004 R14: 0000000000000004 R15: 0000004dc63fe0b6
[  334.041964]  cpuidle_enter+0x4f/0xb0
[  334.041977]  call_cpuidle+0x47/0xd0
[  334.041991]  do_idle+0x372/0x460
[  334.042006]  ? __pfx_do_idle+0x10/0x10
[  334.042022]  cpu_startup_entry+0x58/0x70
[  334.042036]  start_secondary+0x220/0x2b0
[  334.042048]  ? __pfx_start_secondary+0x10/0x10
[  334.042060]  secondary_startup_64_no_verify+0x18f/0x19b
[  334.042079]  </TASK>
[  334.042089] ================================================================================
[  336.974834] ================================================================================
[  336.974888] UBSAN: shift-out-of-bounds in /home/sandy/workplace/wifiTool/driver/rtl8188eus/hal/phydm/phydm_phystatus.c:1714:69
[  336.974907] shift exponent 63 is too large for 32-bit type 'int'
[  336.974921] CPU: 6 PID: 0 Comm: swapper/6 Tainted: G           OE      6.6.58 #1
[  336.974935] Hardware name: Gigabyte Technology Co., Ltd. B660M GAMING AC DDR4/B660M GAMING AC DDR4, BIOS F4 01/17/2022
[  336.974942] Call Trace:
[  336.974948]  <IRQ>
[  336.974956]  dump_stack_lvl+0x48/0x70
[  336.974981]  dump_stack+0x10/0x20
[  336.974993]  __ubsan_handle_shift_out_of_bounds+0x156/0x310
[  336.975013]  phydm_process_rssi_for_dm+0x57b/0xb00 [8188eu]
[  336.975363]  odm_phy_status_query+0x2bc/0x550 [8188eu]
[  336.975688]  rx_query_phy_status+0x4e5/0x990 [8188eu]
[  336.976068]  ? __pfx_rx_query_phy_status+0x10/0x10 [8188eu]
[  336.976426]  ? __asan_memcpy+0x4e/0x80
[  336.976439]  ? _rtw_memcpy+0x10/0x20 [8188eu]
[  336.976847]  pre_recv_entry+0x77/0x150 [8188eu]
[  336.977211]  recvbuf2recvframe+0x5b2/0x710 [8188eu]
[  336.977567]  usb_recv_tasklet+0x12b/0x230 [8188eu]
[  336.977974]  tasklet_action_common.constprop.0+0x275/0x670
[  336.977993]  tasklet_action+0x22/0x30
[  336.978006]  handle_softirqs+0x192/0x5d0
[  336.978022]  __irq_exit_rcu+0x15c/0x1b0
[  336.978035]  irq_exit_rcu+0xe/0x20
[  336.978048]  common_interrupt+0xa4/0xb0
[  336.978059]  </IRQ>
[  336.978064]  <TASK>
[  336.978070]  asm_common_interrupt+0x27/0x40
[  336.978080] RIP: 0010:cpuidle_enter_state+0x1df/0x520
[  336.978092] Code: 01 73 dd 48 83 c4 28 44 89 f0 5b 41 5c 41 5d 41 5e 41 5f 5d 31 d2 31 c9 31 f6 31 ff 45 31 c0 c3 cc cc cc cc fb 0f 1f 44 00 00 <45> 85 f6 0f 89 06 ff ff ff 48 c7 43 18 00 00 00 00 49 83 fd 09 0f
[  336.978103] RSP: 0018:ffff888100dd7d30 EFLAGS: 00000246
[  336.978116] RAX: 0000000000000000 RBX: ffff88885c34ffe0 RCX: 0000000000000000
[  336.978125] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[  336.978131] RBP: ffff888100dd7d80 R08: 0000000000000000 R09: 0000000000000000
[  336.978138] R10: 0000000000000000 R11: 0000000000000000 R12: ffffffffb50627c0
[  336.978145] R13: 0000000000000004 R14: 0000000000000004 R15: 0000004e75422c67
[  336.978160]  cpuidle_enter+0x4f/0xb0
[  336.978174]  call_cpuidle+0x47/0xd0
[  336.978187]  do_idle+0x372/0x460
[  336.978202]  ? __pfx_do_idle+0x10/0x10
[  336.978219]  cpu_startup_entry+0x58/0x70
[  336.978233]  start_secondary+0x220/0x2b0
[  336.978244]  ? __pfx_start_secondary+0x10/0x10
[  336.978256]  secondary_startup_64_no_verify+0x18f/0x19b
[  336.978275]  </TASK>
[  336.978317] ================================================================================