simd-0178: static syscalls

firedancer-io · Dec 16, 2024 · d9e1c7b · d9e1c7b
1 parent fcacb50
commit d9e1c7b
Show file tree

Hide file tree

Showing 6 changed files with 330 additions and 62 deletions.
diff --git a/src/flamenco/runtime/tests/fd_vm_test.c b/src/flamenco/runtime/tests/fd_vm_test.c
@@ -115,6 +115,11 @@ do{
     ulong mask = (1UL << (max_pc % 64)) - 1UL;
     calldests[ max_pc / 64 ] &= mask;
   }
+  ulong entry_pc = fd_ulong_min( input->vm_ctx.entry_pc, rodata_sz / 8 - 1 );
+  if( input->vm_ctx.sbpf_version >= FD_SBPF_V3 ) {
+    /* in v3 we have to enable the entrypoint */
+    calldests[ entry_pc / 64 ] |= ( 1 << ( entry_pc % 64 ) );
+  }
 
   /* Setup syscalls. Have them all be no-ops */
   fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_new( fd_valloc_malloc( valloc, fd_sbpf_syscalls_align(), fd_sbpf_syscalls_footprint() ) );
@@ -154,7 +159,7 @@ do{
     rodata_sz / 8, /* text_cnt */
     0, /* text_off */
     rodata_sz, /* text_sz */
-    input->vm_ctx.entry_pc,
+    entry_pc,
     calldests,
     input->vm_ctx.sbpf_version,
     syscalls,

diff --git a/src/flamenco/vm/fd_vm.c b/src/flamenco/vm/fd_vm.c
@@ -156,22 +156,29 @@ fd_vm_strerror( int err ) {
 int
 fd_vm_validate( fd_vm_t const * vm ) {
 
+  ulong sbpf_version = vm->sbpf_version;
+
   /* A mapping of all the possible 1-byte sBPF opcodes to their
      validation criteria. */
 
-# define FD_VALID       ((uchar)0) /* Valid opcode */
-# define FD_CHECK_JMP   ((uchar)1) /* Validation should check that the instruction is a valid jump */
-# define FD_CHECK_END   ((uchar)2) /* Validation should check that the instruction is a valid endianness conversion */
-# define FD_CHECK_ST    ((uchar)3) /* Validation should check that the instruction is a valid store */
-# define FD_CHECK_LDQ   ((uchar)4) /* Validation should check that the instruction is a valid load-quad */
-# define FD_CHECK_DIV   ((uchar)5) /* Validation should check that the instruction is a valid division by immediate */
-# define FD_CHECK_SH32  ((uchar)6) /* Validation should check that the immediate is a valid 32-bit shift exponent */
-# define FD_CHECK_SH64  ((uchar)7) /* Validation should check that the immediate is a valid 64-bit shift exponent */
-# define FD_INVALID     ((uchar)8) /* The opcode is invalid */
-# define FD_CHECK_CALLX ((uchar)9) /* Validation should check that callx has valid register number */
-# define FD_CHECK_CALLX_DEPR ((uchar)10) /* Older / deprecated FD_CHECK_CALLX */
-
-  static uchar validation_map[ 256 ] = {
+# define FD_VALID               ((uchar)0) /* Valid opcode */
+# define FD_CHECK_JMP_V3        ((uchar)1) /* Validation should check that the instruction is a valid jump (v3+) */
+# define FD_CHECK_END           ((uchar)2) /* Validation should check that the instruction is a valid endianness conversion */
+# define FD_CHECK_ST            ((uchar)3) /* Validation should check that the instruction is a valid store */
+# define FD_CHECK_LDQ           ((uchar)4) /* Validation should check that the instruction is a valid load-quad */
+# define FD_CHECK_DIV           ((uchar)5) /* Validation should check that the instruction is a valid division by immediate */
+# define FD_CHECK_SH32          ((uchar)6) /* Validation should check that the immediate is a valid 32-bit shift exponent */
+# define FD_CHECK_SH64          ((uchar)7) /* Validation should check that the immediate is a valid 64-bit shift exponent */
+# define FD_INVALID             ((uchar)8) /* The opcode is invalid */
+# define FD_CHECK_CALL_REG      ((uchar)9) /* Validation should check that callx has valid register number */
+# define FD_CHECK_CALL_REG_DEPR ((uchar)10) /* Older / deprecated FD_CHECK_CALLX */
+# define FD_CHECK_CALL_IMM      ((uchar)11) /* Check call against functions registry */
+# define FD_CHECK_SYSCALL       ((uchar)12) /* Check call against syscalls registry */
+# define FD_CHECK_JMP_V0        ((uchar)13) /* Validation should check that the instruction is a valid jump (v0..v2) */
+
+  uchar FD_CHECK_JMP = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? FD_CHECK_JMP_V3 : FD_CHECK_JMP_V0;
+
+  uchar validation_map[ 256 ] = {
     /* 0x00 */ FD_INVALID,    /* 0x01 */ FD_INVALID,    /* 0x02 */ FD_INVALID,    /* 0x03 */ FD_INVALID,
     /* 0x04 */ FD_VALID,      /* 0x05 */ FD_CHECK_JMP,  /* 0x06 */ FD_INVALID,    /* 0x07 */ FD_VALID,
     /* 0x08 */ FD_INVALID,    /* 0x09 */ FD_INVALID,    /* 0x0a */ FD_INVALID,    /* 0x0b */ FD_INVALID,
@@ -205,13 +212,13 @@ fd_vm_validate( fd_vm_t const * vm ) {
     /* 0x78 */ FD_INVALID,    /* 0x79 */ FD_INVALID,    /* 0x7a */ FD_INVALID,    /* 0x7b */ FD_INVALID,
     /* 0x7c */ FD_VALID,      /* 0x7d */ FD_CHECK_JMP,  /* 0x7e */ FD_VALID,      /* 0x7f */ FD_VALID,
     /* 0x80 */ FD_INVALID,    /* 0x81 */ FD_INVALID,    /* 0x82 */ FD_INVALID,    /* 0x83 */ FD_INVALID,
-    /* 0x84 */ FD_INVALID,    /* 0x85 */ FD_VALID,      /* 0x86 */ FD_VALID,      /* 0x87 */ FD_CHECK_ST,
+    /* 0x84 */ FD_INVALID,    /* 0x85 */ FD_CHECK_CALL_IMM,/*0x86*/FD_VALID,      /* 0x87 */ FD_CHECK_ST,
     /* 0x88 */ FD_INVALID,    /* 0x89 */ FD_INVALID,    /* 0x8a */ FD_INVALID,    /* 0x8b */ FD_INVALID,
-    /* 0x8c */ FD_VALID,      /* 0x8d */ FD_CHECK_CALLX,/* 0x8e */ FD_VALID,      /* 0x8f */ FD_CHECK_ST,
+    /* 0x8c */ FD_VALID,      /* 0x8d */ FD_CHECK_CALL_REG,/*0x8e*/FD_VALID,      /* 0x8f */ FD_CHECK_ST,
     /* 0x90 */ FD_INVALID,    /* 0x91 */ FD_INVALID,    /* 0x92 */ FD_INVALID,    /* 0x93 */ FD_INVALID,
-    /* 0x94 */ FD_INVALID,    /* 0x95 */ FD_VALID,      /* 0x96 */ FD_VALID,      /* 0x97 */ FD_CHECK_ST,
+    /* 0x94 */ FD_INVALID,    /* 0x95 */ FD_CHECK_SYSCALL,/*0x96*/ FD_VALID,      /* 0x97 */ FD_CHECK_ST,
     /* 0x98 */ FD_INVALID,    /* 0x99 */ FD_INVALID,    /* 0x9a */ FD_INVALID,    /* 0x9b */ FD_INVALID,
-    /* 0x9c */ FD_VALID,      /* 0x9d */ FD_INVALID,    /* 0x9e */ FD_VALID,      /* 0x9f */ FD_CHECK_ST,
+    /* 0x9c */ FD_VALID,      /* 0x9d */ FD_VALID,      /* 0x9e */ FD_VALID,      /* 0x9f */ FD_CHECK_ST,
     /* 0xa0 */ FD_INVALID,    /* 0xa1 */ FD_INVALID,    /* 0xa2 */ FD_INVALID,    /* 0xa3 */ FD_INVALID,
     /* 0xa4 */ FD_VALID,      /* 0xa5 */ FD_CHECK_JMP,  /* 0xa6 */ FD_INVALID,    /* 0xa7 */ FD_VALID,
     /* 0xa8 */ FD_INVALID,    /* 0xa9 */ FD_INVALID,    /* 0xaa */ FD_INVALID,    /* 0xab */ FD_INVALID,
@@ -238,8 +245,6 @@ fd_vm_validate( fd_vm_t const * vm ) {
     /* 0xfc */ FD_INVALID,    /* 0xfd */ FD_INVALID,    /* 0xfe */ FD_VALID,      /* 0xff */ FD_INVALID,
   };
 
-  ulong sbpf_version = vm->sbpf_version;
-
   /* SIMD-0173: LDDW */
   validation_map[ 0x18 ] = FD_VM_SBPF_ENABLE_LDDW(sbpf_version) ? FD_CHECK_LDQ : FD_INVALID;
   validation_map[ 0xf7 ] = FD_VM_SBPF_ENABLE_LDDW(sbpf_version) ? FD_INVALID   : FD_VALID; /* HOR64 */
@@ -280,7 +285,7 @@ fd_vm_validate( fd_vm_t const * vm ) {
   validation_map[ 0x9f ] = FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(sbpf_version) ? FD_CHECK_ST  : FD_VALID;
 
   /* SIMD-0173: CALLX */
-  validation_map[ 0x8d ] = FD_VM_SBPF_CALLX_USES_SRC_REG(sbpf_version) ? FD_CHECK_CALLX : FD_CHECK_CALLX_DEPR;
+  validation_map[ 0x8d ] = FD_VM_SBPF_CALLX_USES_SRC_REG(sbpf_version) ? FD_CHECK_CALL_REG : FD_CHECK_CALL_REG_DEPR;
 
   /* SIMD-0174: MUL, DIV, MOD */
   validation_map[ 0x24 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? FD_INVALID : FD_VALID;
@@ -318,6 +323,11 @@ fd_vm_validate( fd_vm_t const * vm ) {
   validation_map[ 0xf6 ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? FD_CHECK_DIV : FD_INVALID; /* SREM64 */
   validation_map[ 0xfe ] = FD_VM_SBPF_ENABLE_PQR (sbpf_version) ? FD_VALID     : FD_INVALID;
 
+  /* SIMD-0178: static syscalls */
+  validation_map[ 0x85 ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? FD_CHECK_CALL_IMM : FD_VALID;
+  validation_map[ 0x95 ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? FD_CHECK_SYSCALL : FD_VALID;
+  validation_map[ 0x9d ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? FD_VALID : FD_INVALID;
+
   /* FIXME: These checks are not necessary assuming fd_vm_t is populated by metadata
      generated in fd_sbpf_elf_peek (which performs these checks). But there is no guarantee, and
      this non-guarantee is (rightfully) exploited by the fuzz harnesses.
@@ -345,20 +355,31 @@ fd_vm_validate( fd_vm_t const * vm ) {
 
     case FD_VALID: break;
 
-    case FD_CHECK_JMP: {
+    /* Store ops are special because they allow dreg==r10.
+       We use a special validation_code, used later in the
+       "Check registers" section.
+       But there's nothing to do at this time. */
+    case FD_CHECK_ST: break;
+
+    case FD_CHECK_JMP_V0: {
       long jmp_dst = (long)i + (long)instr.offset + 1L;
       if( FD_UNLIKELY( (jmp_dst<0) | (jmp_dst>=(long)text_cnt)                          ) ) return FD_VM_ERR_JMP_OUT_OF_BOUNDS;
+      //FIXME: this shouldn't be here?
       if( FD_UNLIKELY( fd_sbpf_instr( text[ jmp_dst ] ).opcode.raw==FD_SBPF_OP_ADDL_IMM ) ) return FD_VM_ERR_JMP_TO_ADDL_IMM;
       break;
     }
 
+    case FD_CHECK_JMP_V3: {
+      long jmp_dst = (long)i + (long)instr.offset + 1L;
+      if( FD_UNLIKELY( (jmp_dst<0) | (jmp_dst>=(long)text_cnt) ) ) return FD_VM_ERR_JMP_OUT_OF_BOUNDS;
+      break;
+    }
+
     case FD_CHECK_END: {
       if( FD_UNLIKELY( !((instr.imm==16) | (instr.imm==32) | (instr.imm==64)) ) ) return FD_VM_ERR_INVALID_END_IMM;
       break;
     }
 
-    case FD_CHECK_ST: break; /* FIXME: HMMM ... */
-
     /* https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/verifier.rs#L244 */
     case FD_CHECK_LDQ: {
       /* https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/verifier.rs#L131 */
@@ -376,7 +397,7 @@ fd_vm_validate( fd_vm_t const * vm ) {
     }
 
     case FD_CHECK_DIV: {
-      if( FD_UNLIKELY( instr.imm==0 ) ) return FD_VM_ERR_SIGFPE;  /* FIXME: SIGILL? */
+      if( FD_UNLIKELY( instr.imm==0 ) ) return FD_VM_ERR_SIGFPE;
       break;
     }
 
@@ -391,19 +412,44 @@ fd_vm_validate( fd_vm_t const * vm ) {
     }
 
     /* https://github.com/solana-labs/rbpf/blob/v0.8.5/src/verifier.rs#L207 */
-    case FD_CHECK_CALLX: {
+    case FD_CHECK_CALL_REG: {
       if( FD_UNLIKELY( instr.src_reg > 9 ) ) {
         return FD_VM_ERR_INVALID_REG;
       }
       break;
     }
-    case FD_CHECK_CALLX_DEPR: {
+    case FD_CHECK_CALL_REG_DEPR: {
       if( FD_UNLIKELY( instr.imm > 9 ) ) {
         return FD_VM_ERR_INVALID_REG;
       }
       break;
     }
 
+    /* https://github.com/solana-labs/rbpf/blob/4ad935be/src/verifier.rs#L411-L418 */
+    case FD_CHECK_CALL_IMM: {
+      ulong target_pc = (ulong)( (long)i + (long)(int)instr.imm + 1 );
+      if( FD_UNLIKELY( target_pc>text_cnt || !fd_sbpf_calldests_test( vm->calldests, target_pc ) ) ) {
+        return FD_VM_INVALID_FUNCTION;
+      }
+      break;
+    }
+
+    /* https://github.com/solana-labs/rbpf/blob/4ad935be/src/verifier.rs#L423-L428 */
+    case FD_CHECK_SYSCALL: {
+      uint imm = instr.imm;
+      /* check out of bound */
+      if( FD_UNLIKELY( imm==0 || imm >= FD_VM_SBPF_STATIC_SYSCALLS_LIST_SZ ) ) {
+        return FD_VM_INVALID_SYSCALL;
+      }
+      uint syscall_key = FD_VM_SBPF_STATIC_SYSCALLS_LIST[ imm ];
+      /* check active syscall */
+      fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( vm->syscalls, syscall_key, NULL );
+      if( FD_UNLIKELY( !syscall ) ) {
+        return FD_VM_INVALID_SYSCALL;
+      }
+      break;
+    }
+
     case FD_INVALID: default: return FD_VM_ERR_INVALID_OPCODE;
     }
 

diff --git a/src/flamenco/vm/fd_vm_base.h b/src/flamenco/vm/fd_vm_base.h
@@ -75,6 +75,8 @@
 #define FD_VM_ERR_BAD_TEXT          (-36) /* detected a bad text section (overflow, outside rodata boundary, etc.,)*/
 #define FD_VM_SH_OVERFLOW           (-37) /* detected a shift overflow, equivalent to VeriferError::ShiftWithOverflow */
 #define FD_VM_TEXT_SZ_UNALIGNED     (-38) /* detected a text section that is not a multiple of 8 */
+#define FD_VM_INVALID_FUNCTION      (-39) /* detected a text section that is not a multiple of 8 */
+#define FD_VM_INVALID_SYSCALL       (-39) /* detected a text section that is not a multiple of 8 */
 
 /* Syscall Errors
    https://github.com/anza-xyz/agave/blob/v2.0.7/programs/bpf_loader/src/syscalls/mod.rs#L81 */

diff --git a/src/flamenco/vm/fd_vm_interp_core.c b/src/flamenco/vm/fd_vm_interp_core.c
@@ -116,6 +116,11 @@
   interp_jump_table[ 0x14 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(sbpf_version) ? &&interp_0x14 : &&interp_0x14depr;
   interp_jump_table[ 0x17 ] = FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(sbpf_version) ? &&interp_0x17 : &&interp_0x17depr;
 
+  /* SIMD-0178: static syscalls */
+  interp_jump_table[ 0x85 ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? &&interp_0x85 : &&interp_0x85depr;
+  interp_jump_table[ 0x95 ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? &&interp_0x95 : &&interp_0x9d;
+  interp_jump_table[ 0x9d ] = FD_VM_SBPF_STATIC_SYSCALLS (sbpf_version) ? &&interp_0x9d : &&sigill;
+
   /* Unpack the VM state */
 
   ulong pc        = vm->pc;
@@ -657,6 +662,12 @@
   FD_VM_INTERP_INSTR_END;
 
   FD_VM_INTERP_BRANCH_BEGIN(0x85) /* FD_SBPF_OP_CALL_IMM */
+    /* imm has already been validated */
+    FD_VM_INTERP_STACK_PUSH;
+    pc = (ulong)( (long)pc + (long)(int)imm );
+  FD_VM_INTERP_BRANCH_END;
+
+  FD_VM_INTERP_BRANCH_BEGIN(0x85depr) { /* FD_SBPF_OP_CALL_IMM */
 
     fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, imm, NULL );
     if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
@@ -682,22 +693,19 @@
           is a best-effort implementation that should match the VM state
           in all ways except error code. */
 
-      /* Note the original implementation had the imm magic number
-          check after the calldest check.  But fd_pchash_inverse of the
-          magic number is 0xb00c380U.  This is beyond possible text_cnt
-          values.  So we do it first to simplify the code and clean up
-          fault handling. */
-
-      if( FD_UNLIKELY( imm==0x71e3cf81U ) ){
+      /* Special case to handle entrypoint.
+         ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
+         fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
+      if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
         FD_VM_INTERP_STACK_PUSH;
-        pc = entry_pc; /* FIXME: MAGIC NUMBER */
+        pc = entry_pc - 1;
       } else {
-        
+
         ulong target_pc = (ulong)fd_pchash_inverse( imm );
         if( FD_UNLIKELY( target_pc>text_cnt ) ){
           /* ...to match state of Agave VM when faulting
-             Note: this check MUST be BEFORE fd_sbpf_calldests_test,
-             because it prevents overflowing calldests. */
+              Note: this check MUST be BEFORE fd_sbpf_calldests_test,
+              because it prevents overflowing calldests. */
           FD_VM_INTERP_STACK_PUSH;
           goto sigtextbr;
         }
@@ -707,9 +715,8 @@
         }
 
         FD_VM_INTERP_STACK_PUSH;
-        pc = target_pc;
+        pc = target_pc - 1;
       }
-      pc--;
 
     } else {
 
@@ -773,8 +780,7 @@
 
       /* At this point, cu is positive and err is clear */
     }
-
-  FD_VM_INTERP_BRANCH_END;
+  } FD_VM_INTERP_BRANCH_END;
 
   FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
     reg[ dst ] = (ulong)( (uint)reg_dst * imm );
@@ -861,21 +867,70 @@
     reg[ dst ] = (ulong)( (uint)reg_dst % imm );
   FD_VM_INTERP_INSTR_END;
 
-  FD_VM_INTERP_BRANCH_BEGIN(0x95) /* FD_SBPF_OP_EXIT */
-      /* Agave JIT VM exit implementation analysis below.
-
-       Agave references:
-       https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
-       https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
-    if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
-    frame_cnt--;
-    reg[6]   = shadow[ frame_cnt ].r6;
-    reg[7]   = shadow[ frame_cnt ].r7;
-    reg[8]   = shadow[ frame_cnt ].r8;
-    reg[9]   = shadow[ frame_cnt ].r9;
-    reg[10]  = shadow[ frame_cnt ].r10;
-    pc       = shadow[ frame_cnt ].pc;
-  FD_VM_INTERP_BRANCH_END;
+  FD_VM_INTERP_BRANCH_BEGIN(0x95) { /* FD_SBPF_OP_CALL_IMM */
+    /* imm has already been validated to not overflow */
+    uint syscall_key = FD_VM_SBPF_STATIC_SYSCALLS_LIST[ imm ];
+    fd_sbpf_syscalls_t const * syscall = fd_sbpf_syscalls_query_const( syscalls, syscall_key, NULL );
+
+    /* this check is probably useless, as validation includes checking that the
+       syscall is active in this epoch.
+       However, it's safe to keep it here, because at the time of writing this
+       code we're not (re)validating all programs at every new epoch. */
+    if( FD_UNLIKELY( !syscall ) ) goto sigill;
+
+    /* Update the vm with the current vm execution state for the
+       syscall.  Note that BRANCH_BEGIN has pc at the syscall and
+       already updated ic and cu to reflect all instructions up to
+       and including the syscall instruction itself. */
+
+    vm->pc        = pc;
+    vm->ic        = ic;
+    vm->cu        = cu;
+    vm->frame_cnt = frame_cnt;
+
+    /* Do the syscall.  We use ret reduce the risk of the syscall
+       accidentally modifying other registers (note however since a
+       syscall has the vm handle it still do arbitrary modifications
+       to the vm state) and the risk of a pointer escape on reg from
+       inhibiting compiler optimizations (this risk is likely low in
+       as this is the only point in the whole interpreter core that
+       calls outside this translation unit). */
+
+    ulong ret[1];
+    err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret );
+    reg[0] = ret[0];
+
+    /* If we trust syscall implementations to handle the vm state
+       correctly, the below could be implemented as unpacking the vm
+       state and jumping to sigsys on error.  But we provide some
+       extra protection to make various strong guarantees:
+
+       - We do not let the syscall modify pc currently as nothing
+         requires this and it reduces risk of a syscall bug mucking
+         up the interpreter.  If there ever was a syscall that
+         needed to modify the pc (e.g. a syscall that has execution
+         resume from a different location than the instruction
+         following the syscall), do "pc = vm->pc" below.
+
+       - We do not let the syscall modify ic currently as nothing
+         requires this and it keeps the ic precise.  If a future
+         syscall needs this, do "ic = vm->ic" below.
+
+       - We do not let the syscall increase cu as nothing requires
+         this and it guarantees the interpreter will halt in a
+         reasonable finite amount of time.  If a future syscall
+         needs this, do "cu = vm->cu" below.
+
+       - A syscall that returns SIGCOST is always treated as though
+         it also zerod cu. */
+
+    ulong cu_req = vm->cu;
+    cu = fd_ulong_min( cu_req, cu );
+    if( FD_UNLIKELY( err ) ) {
+      if( err==FD_VM_ERR_SIGCOST ) cu = 0UL; /* cmov */
+      goto sigsyscall;
+    }
+  } FD_VM_INTERP_BRANCH_END;
 
   FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
     reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
@@ -901,6 +956,22 @@
   }
   FD_VM_INTERP_INSTR_END;
 
+  FD_VM_INTERP_BRANCH_BEGIN(0x9d) /* FD_SBPF_OP_EXIT */
+      /* Agave JIT VM exit implementation analysis below.
+
+       Agave references:
+       https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
+       https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
+    if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
+    frame_cnt--;
+    reg[6]   = shadow[ frame_cnt ].r6;
+    reg[7]   = shadow[ frame_cnt ].r7;
+    reg[8]   = shadow[ frame_cnt ].r8;
+    reg[9]   = shadow[ frame_cnt ].r9;
+    reg[10]  = shadow[ frame_cnt ].r10;
+    pc       = shadow[ frame_cnt ].pc;
+  FD_VM_INTERP_BRANCH_END;
+
   FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
     reg[ dst ] = reg_dst * reg_src;
   FD_VM_INTERP_INSTR_END;