diff --git a/ceno_zkvm/src/expression.rs b/ceno_zkvm/src/expression.rs index 9d65177e0..bad41b1b4 100644 --- a/ceno_zkvm/src/expression.rs +++ b/ceno_zkvm/src/expression.rs @@ -4,7 +4,6 @@ use std::{ cmp::max, fmt::Display, iter::{Product, Sum}, - mem::MaybeUninit, ops::{Add, AddAssign, Deref, Mul, MulAssign, Neg, Shl, ShlAssign, Sub, SubAssign}, }; @@ -756,12 +755,8 @@ impl WitIn { ) } - pub fn assign( - &self, - instance: &mut [MaybeUninit], - value: E::BaseField, - ) { - instance[self.id as usize] = MaybeUninit::new(value); + pub fn assign(&self, instance: &mut [E::BaseField], value: E::BaseField) { + instance[self.id as usize] = value; } } diff --git a/ceno_zkvm/src/gadgets/div.rs b/ceno_zkvm/src/gadgets/div.rs index c9d86ab87..95b7cc49f 100644 --- a/ceno_zkvm/src/gadgets/div.rs +++ b/ceno_zkvm/src/gadgets/div.rs @@ -1,4 +1,4 @@ -use std::{fmt::Display, mem::MaybeUninit}; +use std::fmt::Display; use ff_ext::ExtensionField; @@ -53,7 +53,7 @@ impl DivConfig { pub fn assign_instance<'a>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, divisor: &Value<'a, u32>, quotient: &Value<'a, u32>, diff --git a/ceno_zkvm/src/gadgets/is_lt.rs b/ceno_zkvm/src/gadgets/is_lt.rs index 3885b7257..967efc868 100644 --- a/ceno_zkvm/src/gadgets/is_lt.rs +++ b/ceno_zkvm/src/gadgets/is_lt.rs @@ -1,4 +1,4 @@ -use std::{fmt::Display, mem::MaybeUninit}; +use std::fmt::Display; use ceno_emul::{SWord, Word}; use ff_ext::ExtensionField; @@ -52,7 +52,7 @@ impl AssertLTConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: u64, rhs: u64, @@ -106,7 +106,7 @@ impl IsLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: u64, rhs: u64, @@ -118,7 +118,7 @@ impl IsLtConfig { pub fn assign_instance_signed( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -184,7 +184,7 @@ impl InnerLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: u64, rhs: u64, @@ -202,7 +202,7 @@ impl InnerLtConfig { // TODO: refactor with the above function pub fn assign_instance_signed( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -256,7 +256,7 @@ impl AssertSignedLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -299,7 +299,7 @@ impl SignedLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, @@ -351,7 +351,7 @@ impl InnerSignedLtConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, lhs: SWord, rhs: SWord, diff --git a/ceno_zkvm/src/gadgets/is_zero.rs b/ceno_zkvm/src/gadgets/is_zero.rs index f7d749354..e156cb9ce 100644 --- a/ceno_zkvm/src/gadgets/is_zero.rs +++ b/ceno_zkvm/src/gadgets/is_zero.rs @@ -1,4 +1,3 @@ -use std::mem::MaybeUninit; use ff_ext::ExtensionField; use goldilocks::SmallField; @@ -64,7 +63,7 @@ impl IsZeroConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], x: F, ) -> Result<(), ZKVMError> { let (is_zero, inverse) = if x.is_zero_vartime() { @@ -117,7 +116,7 @@ impl IsEqualConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], a: F, b: F, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/gadgets/signed_ext.rs b/ceno_zkvm/src/gadgets/signed_ext.rs index d1dc8ed62..3b6757bc3 100644 --- a/ceno_zkvm/src/gadgets/signed_ext.rs +++ b/ceno_zkvm/src/gadgets/signed_ext.rs @@ -7,7 +7,7 @@ use crate::{ witness::LkMultiplicity, }; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; #[derive(Debug)] pub struct SignedExtendConfig { @@ -84,7 +84,7 @@ impl SignedExtendConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, val: u64, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions.rs b/ceno_zkvm/src/instructions.rs index e87675dd6..a663c6f77 100644 --- a/ceno_zkvm/src/instructions.rs +++ b/ceno_zkvm/src/instructions.rs @@ -1,5 +1,3 @@ -use std::mem::MaybeUninit; - use ceno_emul::StepRecord; use ff_ext::ExtensionField; use rayon::{ @@ -10,13 +8,12 @@ use rayon::{ use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, - scheme::constants::MIN_PAR_SIZE, witness::{LkMultiplicity, RowMajorMatrix}, }; -use ff::Field; pub mod riscv; +#[derive(Clone)] pub enum InstancePaddingStrategy { Zero, RepeatLast, @@ -37,7 +34,7 @@ pub trait Instruction { // assign single instance giving step from trace fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError>; @@ -56,7 +53,9 @@ pub trait Instruction { } .max(1); let lk_multiplicity = LkMultiplicity::default(); - let mut raw_witin = RowMajorMatrix::::new(steps.len(), num_witin); + dbg!(steps.len(), num_witin); + let mut raw_witin = + RowMajorMatrix::::new(steps.len(), num_witin, Self::padding_strategy()); let raw_witin_iter = raw_witin.par_batch_iter_mut(num_instance_per_batch); raw_witin_iter @@ -73,35 +72,6 @@ pub trait Instruction { }) .collect::>()?; - let num_padding_instances = raw_witin.num_padding_instances(); - if num_padding_instances > 0 { - // Fill the padding based on strategy - - let padding_instance = match Self::padding_strategy() { - InstancePaddingStrategy::Zero => { - vec![MaybeUninit::new(E::BaseField::ZERO); num_witin] - } - InstancePaddingStrategy::RepeatLast if steps.is_empty() => { - tracing::debug!("No {} steps to repeat, using zero padding", Self::name()); - vec![MaybeUninit::new(E::BaseField::ZERO); num_witin] - } - InstancePaddingStrategy::RepeatLast => raw_witin[steps.len() - 1].to_vec(), - }; - - let num_padding_instance_per_batch = if num_padding_instances > 256 { - num_padding_instances.div_ceil(nthreads) - } else { - num_padding_instances - }; - raw_witin - .par_batch_iter_padding_mut(None, num_padding_instance_per_batch) - .with_min_len(MIN_PAR_SIZE) - .for_each(|row| { - row.chunks_mut(num_witin) - .for_each(|instance| instance.copy_from_slice(padding_instance.as_slice())); - }); - } - Ok((raw_witin, lk_multiplicity)) } } diff --git a/ceno_zkvm/src/instructions/riscv/arith.rs b/ceno_zkvm/src/instructions/riscv/arith.rs index c1d9d4412..2ea302844 100644 --- a/ceno_zkvm/src/instructions/riscv/arith.rs +++ b/ceno_zkvm/src/instructions/riscv/arith.rs @@ -11,7 +11,6 @@ use crate::{ uint::{Value, ValueMul}, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles R-Instructions that represent registers values as 2 * u16. #[derive(Debug)] @@ -106,7 +105,7 @@ impl Instruction for ArithInstruction::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/arith_imm.rs b/ceno_zkvm/src/instructions/riscv/arith_imm.rs index f5aed7f0f..c4c09629b 100644 --- a/ceno_zkvm/src/instructions/riscv/arith_imm.rs +++ b/ceno_zkvm/src/instructions/riscv/arith_imm.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::StepRecord; use ff_ext::ExtensionField; @@ -57,7 +57,7 @@ impl Instruction for AddiInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/b_insn.rs b/ceno_zkvm/src/instructions/riscv/b_insn.rs index c39a1babe..646e9bd05 100644 --- a/ceno_zkvm/src/instructions/riscv/b_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/b_insn.rs @@ -13,7 +13,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; // Opcode: 1100011 // Funct3: @@ -88,7 +87,7 @@ impl BInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs b/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs index 8c82d6834..b8c96fbd1 100644 --- a/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/branch/beq_circuit.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; @@ -72,7 +72,7 @@ impl Instruction for BeqCircuit { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/branch/blt.rs b/ceno_zkvm/src/instructions/riscv/branch/blt.rs index c5e0798f2..dfe5995e1 100644 --- a/ceno_zkvm/src/instructions/riscv/branch/blt.rs +++ b/ceno_zkvm/src/instructions/riscv/branch/blt.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ff_ext::ExtensionField; @@ -66,7 +66,7 @@ impl Instruction for BltCircuit { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/branch/bltu.rs b/ceno_zkvm/src/instructions/riscv/branch/bltu.rs index 896bf19da..713158fdc 100644 --- a/ceno_zkvm/src/instructions/riscv/branch/bltu.rs +++ b/ceno_zkvm/src/instructions/riscv/branch/bltu.rs @@ -75,7 +75,7 @@ impl Instruction for BltuCircuit fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [std::mem::MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/config.rs b/ceno_zkvm/src/instructions/riscv/config.rs index cd13e1148..e35465add 100644 --- a/ceno_zkvm/src/instructions/riscv/config.rs +++ b/ceno_zkvm/src/instructions/riscv/config.rs @@ -1,4 +1,3 @@ -use std::mem::MaybeUninit; use crate::{expression::WitIn, set_val, utils::i64_to_base, witness::LkMultiplicity}; use goldilocks::SmallField; @@ -25,7 +24,7 @@ pub struct MsbInput<'a> { impl MsbInput<'_> { pub fn assign( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], config: &MsbConfig, lk_multiplicity: &mut LkMultiplicity, ) -> (u8, u8) { @@ -61,7 +60,7 @@ pub struct UIntLtuInput<'a> { impl UIntLtuInput<'_> { pub fn assign( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], config: &UIntLtuConfig, lk_multiplicity: &mut LkMultiplicity, ) -> bool { @@ -138,7 +137,7 @@ pub struct UIntLtInput<'a> { impl UIntLtInput<'_> { pub fn assign( &self, - instance: &mut [MaybeUninit], + instance: &mut [F], config: &UIntLtConfig, lk_multiplicity: &mut LkMultiplicity, ) -> bool { diff --git a/ceno_zkvm/src/instructions/riscv/divu.rs b/ceno_zkvm/src/instructions/riscv/divu.rs index 8d25f583f..0b44a85a0 100644 --- a/ceno_zkvm/src/instructions/riscv/divu.rs +++ b/ceno_zkvm/src/instructions/riscv/divu.rs @@ -16,7 +16,6 @@ use crate::{ uint::Value, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; use std::marker::PhantomData; pub struct ArithConfig { @@ -124,7 +123,7 @@ impl Instruction for ArithInstruction], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -173,7 +172,6 @@ mod test { use ceno_emul::{Change, InsnKind, StepRecord, Word, encode_rv32}; use goldilocks::GoldilocksExt2; use itertools::Itertools; - use multilinear_extensions::mle::IntoMLEs; use rand::Rng; use crate::{ @@ -238,7 +236,6 @@ mod test { MockProver::assert_with_expected_errors( &cb, &raw_witin - .de_interleaving() .into_mles() .into_iter() .map(|v| v.into()) diff --git a/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs b/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs index 1fa7dc4c2..1d98d1271 100644 --- a/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/dummy/dummy_circuit.rs @@ -19,7 +19,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// DummyInstruction can handle any instruction and produce its side-effects. pub struct DummyInstruction(PhantomData<(E, I)>); @@ -69,7 +68,7 @@ impl Instruction for DummyInstruction::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -214,7 +213,7 @@ impl DummyConfig { fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/ecall/halt.rs b/ceno_zkvm/src/instructions/riscv/ecall/halt.rs index 47ed7b9b7..95dba096d 100644 --- a/ceno_zkvm/src/instructions/riscv/ecall/halt.rs +++ b/ceno_zkvm/src/instructions/riscv/ecall/halt.rs @@ -16,7 +16,7 @@ use crate::{ }; use ceno_emul::{StepRecord, Tracer}; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct HaltConfig { ecall_cfg: EcallInstructionConfig, @@ -65,7 +65,7 @@ impl Instruction for HaltInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/ecall_insn.rs b/ceno_zkvm/src/instructions/riscv/ecall_insn.rs index 3bd2faa1e..123667ed1 100644 --- a/ceno_zkvm/src/instructions/riscv/ecall_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/ecall_insn.rs @@ -12,7 +12,6 @@ use crate::{ }; use ceno_emul::{InsnKind::EANY, PC_STEP_SIZE, Platform, StepRecord, Tracer}; use ff_ext::ExtensionField; -use std::mem::MaybeUninit; pub struct EcallInstructionConfig { pub pc: WitIn, @@ -68,7 +67,7 @@ impl EcallInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/i_insn.rs b/ceno_zkvm/src/instructions/riscv/i_insn.rs index 76fb2478c..52147cdec 100644 --- a/ceno_zkvm/src/instructions/riscv/i_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/i_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles the common part of I-type instructions: /// - PC, cycle, fetch. @@ -57,7 +56,7 @@ impl IInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/im_insn.rs b/ceno_zkvm/src/instructions/riscv/im_insn.rs index 82ecf22ce..36ad1a8a1 100644 --- a/ceno_zkvm/src/instructions/riscv/im_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/im_insn.rs @@ -9,7 +9,6 @@ use crate::{ }; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::mem::MaybeUninit; /// This config handle the common part of I-type Instruction (memory variant) /// - PC, cycle, fetch @@ -62,7 +61,7 @@ impl IMInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/insn_base.rs b/ceno_zkvm/src/instructions/riscv/insn_base.rs index 9588cb34f..abe810c26 100644 --- a/ceno_zkvm/src/instructions/riscv/insn_base.rs +++ b/ceno_zkvm/src/instructions/riscv/insn_base.rs @@ -18,7 +18,6 @@ use crate::{ witness::LkMultiplicity, }; use ceno_emul::Tracer; -use core::mem::MaybeUninit; use std::{iter, marker::PhantomData}; #[derive(Debug)] @@ -59,7 +58,7 @@ impl StateInOut { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], // lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -107,7 +106,7 @@ impl ReadRS1 { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -161,7 +160,7 @@ impl ReadRS2 { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -217,7 +216,7 @@ impl WriteRD { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -275,7 +274,7 @@ impl ReadMEM { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -328,7 +327,7 @@ impl WriteMEM { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -437,7 +436,7 @@ impl MemAddr { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, addr: Word, ) -> Result<(), ZKVMError> { @@ -477,6 +476,7 @@ mod test { ROMType, circuit_builder::{CircuitBuilder, ConstraintSystem}, error::ZKVMError, + instructions::InstancePaddingStrategy, scheme::mock_prover::MockProver, witness::{LkMultiplicity, RowMajorMatrix}, }; @@ -516,7 +516,11 @@ mod test { let mut lkm = LkMultiplicity::default(); let num_rows = 2; - let mut raw_witin = RowMajorMatrix::::new(num_rows, cb.cs.num_witin as usize); + let mut raw_witin = RowMajorMatrix::::new( + num_rows, + cb.cs.num_witin as usize, + InstancePaddingStrategy::Zero, + ); for instance in raw_witin.iter_mut() { mem_addr.assign_instance(instance, &mut lkm, addr)?; } @@ -542,7 +546,6 @@ mod test { MockProver::assert_with_expected_errors( &cb, &raw_witin - .de_interleaving() .into_mles() .into_iter() .map(|v| v.into()) diff --git a/ceno_zkvm/src/instructions/riscv/j_insn.rs b/ceno_zkvm/src/instructions/riscv/j_insn.rs index 8dc65e539..81cc8c493 100644 --- a/ceno_zkvm/src/instructions/riscv/j_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/j_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; // Opcode: 1101111 @@ -53,7 +52,7 @@ impl JInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/auipc.rs b/ceno_zkvm/src/instructions/riscv/jump/auipc.rs index 28be09c66..ed1f07d1f 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/auipc.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/auipc.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::InsnKind; use ff_ext::ExtensionField; @@ -71,7 +71,7 @@ impl Instruction for AuipcInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/jal.rs b/ceno_zkvm/src/instructions/riscv/jump/jal.rs index 44facf944..24bcad463 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/jal.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/jal.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ff_ext::ExtensionField; @@ -62,7 +62,7 @@ impl Instruction for JalInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/jalr.rs b/ceno_zkvm/src/instructions/riscv/jump/jalr.rs index 0339a6b0a..1d9c2c025 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/jalr.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/jalr.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ff::Field; use ff_ext::ExtensionField; @@ -110,7 +110,7 @@ impl Instruction for JalrInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/jump/lui.rs b/ceno_zkvm/src/instructions/riscv/jump/lui.rs index 1ac67f64f..8ad9d497e 100644 --- a/ceno_zkvm/src/instructions/riscv/jump/lui.rs +++ b/ceno_zkvm/src/instructions/riscv/jump/lui.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::InsnKind; use ff_ext::ExtensionField; @@ -48,7 +48,7 @@ impl Instruction for LuiInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs b/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs index 1fbdccbe2..30fe370e0 100644 --- a/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/logic/logic_circuit.rs @@ -1,6 +1,5 @@ //! The circuit implementation of logic instructions. -use core::mem::MaybeUninit; use ff_ext::ExtensionField; use std::marker::PhantomData; @@ -50,7 +49,7 @@ impl Instruction for LogicInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -102,7 +101,7 @@ impl LogicConfig { fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs b/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs index 4efd100fc..9328e27a7 100644 --- a/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs +++ b/ceno_zkvm/src/instructions/riscv/logic_imm/logic_imm_circuit.rs @@ -1,6 +1,5 @@ //! The circuit implementation of logic instructions. -use core::mem::MaybeUninit; use ff_ext::ExtensionField; use std::marker::PhantomData; @@ -50,7 +49,7 @@ impl Instruction for LogicInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -103,7 +102,7 @@ impl LogicConfig { fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/memory/gadget.rs b/ceno_zkvm/src/instructions/riscv/memory/gadget.rs index 6539c1325..e43d862fe 100644 --- a/ceno_zkvm/src/instructions/riscv/memory/gadget.rs +++ b/ceno_zkvm/src/instructions/riscv/memory/gadget.rs @@ -11,7 +11,6 @@ use ceno_emul::StepRecord; use ff::Field; use ff_ext::ExtensionField; use itertools::izip; -use std::mem::MaybeUninit; pub struct MemWordChange { prev_limb_bytes: Vec, @@ -147,7 +146,7 @@ impl MemWordChange { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, shift: u32, diff --git a/ceno_zkvm/src/instructions/riscv/memory/load.rs b/ceno_zkvm/src/instructions/riscv/memory/load.rs index a06c17687..fff79f2df 100644 --- a/ceno_zkvm/src/instructions/riscv/memory/load.rs +++ b/ceno_zkvm/src/instructions/riscv/memory/load.rs @@ -19,7 +19,7 @@ use crate::{ use ceno_emul::{ByteAddr, InsnKind, StepRecord}; use ff_ext::ExtensionField; use itertools::izip; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct LoadConfig { im_insn: IMInstructionConfig, @@ -199,7 +199,7 @@ impl Instruction for LoadInstruction], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/memory/store.rs b/ceno_zkvm/src/instructions/riscv/memory/store.rs index d1d941b97..29c950957 100644 --- a/ceno_zkvm/src/instructions/riscv/memory/store.rs +++ b/ceno_zkvm/src/instructions/riscv/memory/store.rs @@ -17,7 +17,7 @@ use crate::{ }; use ceno_emul::{ByteAddr, InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct StoreConfig { s_insn: SInstructionConfig, @@ -128,7 +128,7 @@ impl Instruction fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/mulh.rs b/ceno_zkvm/src/instructions/riscv/mulh.rs index be467de8d..7ec706b7b 100644 --- a/ceno_zkvm/src/instructions/riscv/mulh.rs +++ b/ceno_zkvm/src/instructions/riscv/mulh.rs @@ -101,7 +101,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; pub struct MulhInstructionBase(PhantomData<(E, I)>); @@ -267,7 +266,7 @@ impl Instruction for MulhInstructionBas fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { @@ -373,7 +372,7 @@ impl Signed { pub fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, val: &Value, ) -> Result { diff --git a/ceno_zkvm/src/instructions/riscv/r_insn.rs b/ceno_zkvm/src/instructions/riscv/r_insn.rs index 79bf50eb0..78c262290 100644 --- a/ceno_zkvm/src/instructions/riscv/r_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/r_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles the common part of R-type instructions: /// - PC, cycle, fetch. @@ -61,7 +60,7 @@ impl RInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/s_insn.rs b/ceno_zkvm/src/instructions/riscv/s_insn.rs index c328bc3b7..efe271337 100644 --- a/ceno_zkvm/src/instructions/riscv/s_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/s_insn.rs @@ -9,7 +9,6 @@ use crate::{ }; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::mem::MaybeUninit; /// This config handles the common part of S-type instructions: /// - PC, cycle, fetch. @@ -70,7 +69,7 @@ impl SInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/shift.rs b/ceno_zkvm/src/instructions/riscv/shift.rs index 97b82aaf3..792472c4d 100644 --- a/ceno_zkvm/src/instructions/riscv/shift.rs +++ b/ceno_zkvm/src/instructions/riscv/shift.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::InsnKind; use ff_ext::ExtensionField; @@ -163,7 +163,7 @@ impl Instruction for ShiftLogicalInstru fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [std::mem::MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut crate::witness::LkMultiplicity, step: &ceno_emul::StepRecord, ) -> Result<(), crate::error::ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/shift_imm.rs b/ceno_zkvm/src/instructions/riscv/shift_imm.rs index 44cb01288..771121f89 100644 --- a/ceno_zkvm/src/instructions/riscv/shift_imm.rs +++ b/ceno_zkvm/src/instructions/riscv/shift_imm.rs @@ -15,7 +15,7 @@ use crate::{ }; use ceno_emul::{InsnKind, StepRecord}; use ff_ext::ExtensionField; -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; pub struct ShiftImmConfig { i_insn: IInstructionConfig, @@ -144,7 +144,7 @@ impl Instruction for ShiftImmInstructio fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/slt.rs b/ceno_zkvm/src/instructions/riscv/slt.rs index 2f1a3af97..65cca0169 100644 --- a/ceno_zkvm/src/instructions/riscv/slt.rs +++ b/ceno_zkvm/src/instructions/riscv/slt.rs @@ -8,7 +8,6 @@ use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, gadgets::SignedLtConfig, instructions::Instruction, uint::Value, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles R-Instructions that represent registers values as 2 * u16. #[derive(Debug)] @@ -60,7 +59,7 @@ impl Instruction for SltInstruction { fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/slti.rs b/ceno_zkvm/src/instructions/riscv/slti.rs index d88f0336c..6b5097eec 100644 --- a/ceno_zkvm/src/instructions/riscv/slti.rs +++ b/ceno_zkvm/src/instructions/riscv/slti.rs @@ -20,7 +20,6 @@ use crate::{ utils::i64_to_base, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; #[derive(Debug)] pub struct SetLessThanImmConfig { @@ -96,7 +95,7 @@ impl Instruction for SetLessThanImmInst fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/sltu.rs b/ceno_zkvm/src/instructions/riscv/sltu.rs index 26f15b9bc..ecab275de 100644 --- a/ceno_zkvm/src/instructions/riscv/sltu.rs +++ b/ceno_zkvm/src/instructions/riscv/sltu.rs @@ -12,7 +12,6 @@ use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, gadgets::IsLtConfig, instructions::Instruction, uint::Value, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles R-Instructions that represent registers values as 2 * u16. #[derive(Debug)] @@ -78,7 +77,7 @@ impl Instruction for ArithInstruction::BaseField>], + instance: &mut [::BaseField], lkm: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/instructions/riscv/u_insn.rs b/ceno_zkvm/src/instructions/riscv/u_insn.rs index 613ea1dcc..719ee2df1 100644 --- a/ceno_zkvm/src/instructions/riscv/u_insn.rs +++ b/ceno_zkvm/src/instructions/riscv/u_insn.rs @@ -10,7 +10,6 @@ use crate::{ tables::InsnRecord, witness::LkMultiplicity, }; -use core::mem::MaybeUninit; /// This config handles the common part of the U-type instruction: /// - PC, cycle, fetch @@ -51,7 +50,7 @@ impl UInstructionConfig { pub fn assign_instance( &self, - instance: &mut [MaybeUninit<::BaseField>], + instance: &mut [::BaseField], lk_multiplicity: &mut LkMultiplicity, step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/scheme/mock_prover.rs b/ceno_zkvm/src/scheme/mock_prover.rs index 2ee5d1a66..a7a081f29 100644 --- a/ceno_zkvm/src/scheme/mock_prover.rs +++ b/ceno_zkvm/src/scheme/mock_prover.rs @@ -653,10 +653,7 @@ impl<'a, E: ExtensionField + Hash> MockProver { for table_expr in &cs.lk_table_expressions { for row in fixed.iter_rows() { // TODO: Find a better way to obtain the row content. - let row = row - .iter() - .map(|v| unsafe { (*v).assume_init() }.into()) - .collect::>(); + let row = row.iter().map(|v| (*v).into()).collect::>(); let rlc_record = eval_by_expr_with_fixed(&row, &[], &challenge, &table_expr.values); t_vec.push(rlc_record.to_canonical_u64_vec()); } @@ -727,7 +724,6 @@ Hints: lkm: Option, ) { let wits_in = raw_witin - .de_interleaving() .into_mles() .into_iter() .map(|v| v.into()) @@ -1206,7 +1202,6 @@ Hints: #[cfg(test)] mod tests { - use std::mem::MaybeUninit; use super::*; use crate::{ @@ -1214,6 +1209,7 @@ mod tests { error::ZKVMError, expression::{ToExpr, WitIn}, gadgets::{AssertLTConfig, IsLtConfig}, + instructions::InstancePaddingStrategy, set_val, witness::{LkMultiplicity, RowMajorMatrix}, }; @@ -1377,7 +1373,7 @@ mod tests { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], input: AssertLtCircuitInput, lk_multiplicity: &mut LkMultiplicity, ) -> Result<(), ZKVMError> { @@ -1395,7 +1391,11 @@ mod tests { instances: Vec, lk_multiplicity: &mut LkMultiplicity, ) -> Result, ZKVMError> { - let mut raw_witin = RowMajorMatrix::::new(instances.len(), num_witin); + let mut raw_witin = RowMajorMatrix::::new( + instances.len(), + num_witin, + InstancePaddingStrategy::Zero, + ); let raw_witin_iter = raw_witin.iter_mut(); raw_witin_iter @@ -1491,7 +1491,7 @@ mod tests { fn assign_instance( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], input: LtCircuitInput, lk_multiplicity: &mut LkMultiplicity, ) -> Result<(), ZKVMError> { @@ -1509,7 +1509,11 @@ mod tests { instances: Vec, lk_multiplicity: &mut LkMultiplicity, ) -> Result, ZKVMError> { - let mut raw_witin = RowMajorMatrix::::new(instances.len(), num_witin); + let mut raw_witin = RowMajorMatrix::::new( + instances.len(), + num_witin, + InstancePaddingStrategy::Zero, + ); let raw_witin_iter = raw_witin.iter_mut(); raw_witin_iter diff --git a/ceno_zkvm/src/scheme/tests.rs b/ceno_zkvm/src/scheme/tests.rs index c66884ee7..056c9ea54 100644 --- a/ceno_zkvm/src/scheme/tests.rs +++ b/ceno_zkvm/src/scheme/tests.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::{ CENO_PLATFORM, @@ -71,7 +71,7 @@ impl Instruction for Test fn assign_instance( config: &Self::InstructionConfig, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], _lk_multiplicity: &mut LkMultiplicity, _step: &StepRecord, ) -> Result<(), ZKVMError> { diff --git a/ceno_zkvm/src/tables/mod.rs b/ceno_zkvm/src/tables/mod.rs index f498b868e..88a5834de 100644 --- a/ceno_zkvm/src/tables/mod.rs +++ b/ceno_zkvm/src/tables/mod.rs @@ -1,11 +1,9 @@ use crate::{ - circuit_builder::CircuitBuilder, error::ZKVMError, scheme::constants::MIN_PAR_SIZE, + circuit_builder::CircuitBuilder, error::ZKVMError, witness::RowMajorMatrix, }; use ff_ext::ExtensionField; -use goldilocks::SmallField; -use rayon::iter::{IndexedParallelIterator, ParallelIterator}; -use std::{collections::HashMap, mem::MaybeUninit}; +use std::collections::HashMap; mod range; pub use range::*; @@ -41,39 +39,4 @@ pub trait TableCircuit { multiplicity: &[HashMap], input: &Self::WitnessInput, ) -> Result, ZKVMError>; - - fn padding_zero( - table: &mut RowMajorMatrix, - num_witin: usize, - ) -> Result<(), ZKVMError> { - padding_zero(table, num_witin, None); - Ok(()) - } -} - -/// Fill the padding with zeros. Start after the given `num_instances`, or detect it from the table. -pub fn padding_zero( - table: &mut RowMajorMatrix, - num_cols: usize, - num_instances: Option, -) { - // Fill the padding with zeros, if any. - let num_padding_instances = table.num_padding_instances(); - if num_padding_instances > 0 { - let nthreads = - std::env::var("RAYON_NUM_THREADS").map_or(8, |s| s.parse::().unwrap_or(8)); - let padding_instance = vec![MaybeUninit::new(F::ZERO); num_cols]; - let num_padding_instance_per_batch = if num_padding_instances > 256 { - num_padding_instances.div_ceil(nthreads) - } else { - num_padding_instances - }; - table - .par_batch_iter_padding_mut(num_instances, num_padding_instance_per_batch) - .with_min_len(MIN_PAR_SIZE) - .for_each(|row| { - row.chunks_mut(num_cols) - .for_each(|instance| instance.copy_from_slice(padding_instance.as_slice())); - }); - } } diff --git a/ceno_zkvm/src/tables/ops/ops_circuit.rs b/ceno_zkvm/src/tables/ops/ops_circuit.rs index cd48ebe19..988c9eba6 100644 --- a/ceno_zkvm/src/tables/ops/ops_circuit.rs +++ b/ceno_zkvm/src/tables/ops/ops_circuit.rs @@ -51,8 +51,8 @@ impl TableCircuit for OpsTableCircuit num_fixed: usize, _input: &(), ) -> RowMajorMatrix { - let mut table = config.generate_fixed_traces(num_fixed, OP::content()); - Self::padding_zero(&mut table, num_fixed).expect("padding error"); + let table = config.generate_fixed_traces(num_fixed, OP::content()); + // Self::padding_zero(&mut table, num_fixed).expect("padding error"); table } @@ -63,8 +63,8 @@ impl TableCircuit for OpsTableCircuit _input: &(), ) -> Result, ZKVMError> { let multiplicity = &multiplicity[OP::ROM_TYPE as usize]; - let mut table = config.assign_instances(num_witin, multiplicity, OP::len())?; - Self::padding_zero(&mut table, num_witin)?; + let table = config.assign_instances(num_witin, multiplicity, OP::len())?; + // Self::padding_zero(&mut table, num_witin)?; Ok(table) } } diff --git a/ceno_zkvm/src/tables/ops/ops_impl.rs b/ceno_zkvm/src/tables/ops/ops_impl.rs index ddd407078..f233ad8c3 100644 --- a/ceno_zkvm/src/tables/ops/ops_impl.rs +++ b/ceno_zkvm/src/tables/ops/ops_impl.rs @@ -4,12 +4,13 @@ use ff_ext::ExtensionField; use goldilocks::SmallField; use itertools::Itertools; use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; -use std::{collections::HashMap, mem::MaybeUninit}; +use std::collections::HashMap; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, + instructions::InstancePaddingStrategy, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ROMType, @@ -47,7 +48,9 @@ impl OpTableConfig { num_fixed: usize, content: Vec<[u64; 3]>, ) -> RowMajorMatrix { - let mut fixed = RowMajorMatrix::::new(content.len(), num_fixed); + dbg!(content.len(), num_fixed); + let mut fixed = + RowMajorMatrix::::new(content.len(), num_fixed, InstancePaddingStrategy::Zero); fixed .par_iter_mut() @@ -68,7 +71,9 @@ impl OpTableConfig { multiplicity: &HashMap, length: usize, ) -> Result, ZKVMError> { - let mut witness = RowMajorMatrix::::new(length, num_witin); + dbg!(length, num_witin); + let mut witness = + RowMajorMatrix::::new(length, num_witin, InstancePaddingStrategy::Zero); let mut mlts = vec![0; length]; for (idx, mlt) in multiplicity { diff --git a/ceno_zkvm/src/tables/program.rs b/ceno_zkvm/src/tables/program.rs index f1d38ada8..0f1be6558 100644 --- a/ceno_zkvm/src/tables/program.rs +++ b/ceno_zkvm/src/tables/program.rs @@ -1,13 +1,14 @@ -use std::{collections::HashMap, marker::PhantomData, mem::MaybeUninit}; +use std::{collections::HashMap, marker::PhantomData}; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, + instructions::InstancePaddingStrategy, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ROMType, - tables::{TableCircuit, padding_zero}, + tables::TableCircuit, utils::i64_to_base, witness::RowMajorMatrix, }; @@ -158,7 +159,12 @@ impl TableCircuit for ProgramTableCircuit { let pc_base = program.base_address; assert!(num_instructions <= config.program_size); - let mut fixed = RowMajorMatrix::::new(config.program_size, num_fixed); + dbg!(config.program_size, num_fixed); + let mut fixed = RowMajorMatrix::::new( + config.program_size, + num_fixed, + InstancePaddingStrategy::Zero, + ); fixed .par_iter_mut() @@ -167,7 +173,7 @@ impl TableCircuit for ProgramTableCircuit { .for_each(|(row, i)| { let pc = pc_base + (i * PC_STEP_SIZE) as u32; let insn = DecodedInstruction::new(program.instructions[i]); - let values = InsnRecord::from_decoded(pc, &insn); + let values: InsnRecord<_> = InsnRecord::from_decoded(pc, &insn); // Copy all the fields. for (col, val) in config.record.as_slice().iter().zip_eq(values.as_slice()) { @@ -175,9 +181,7 @@ impl TableCircuit for ProgramTableCircuit { } }); - assert_eq!(INVALID as u64, 0, "0 padding must be invalid instructions"); - padding_zero(&mut fixed, num_fixed, Some(num_instructions)); - + // Self::padding_zero(&mut fixed, num_fixed).expect("padding error"); fixed } @@ -195,7 +199,12 @@ impl TableCircuit for ProgramTableCircuit { prog_mlt[i] = *mlt; } - let mut witness = RowMajorMatrix::::new(config.program_size, num_witin); + dbg!(config.program_size, num_witin); + let mut witness = RowMajorMatrix::::new( + config.program_size, + num_witin, + InstancePaddingStrategy::Zero, + ); witness .par_iter_mut() .with_min_len(MIN_PAR_SIZE) @@ -204,8 +213,7 @@ impl TableCircuit for ProgramTableCircuit { set_val!(row, config.mlt, E::BaseField::from(mlt as u64)); }); - padding_zero(&mut witness, num_witin, Some(program.instructions.len())); - + // Self::padding_zero(&mut witness, num_witin).expect("padding error"); Ok(witness) } } diff --git a/ceno_zkvm/src/tables/ram/ram_circuit.rs b/ceno_zkvm/src/tables/ram/ram_circuit.rs index 81f0fcfaa..9ed7f9067 100644 --- a/ceno_zkvm/src/tables/ram/ram_circuit.rs +++ b/ceno_zkvm/src/tables/ram/ram_circuit.rs @@ -6,6 +6,7 @@ use ff_ext::ExtensionField; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, + instructions::InstancePaddingStrategy, structs::{ProgramParams, RAMType}, tables::TableCircuit, witness::RowMajorMatrix, @@ -192,7 +193,7 @@ impl TableC _num_fixed: usize, _init_v: &Self::FixedInput, ) -> RowMajorMatrix { - RowMajorMatrix::::new(0, 0) + RowMajorMatrix::::new(0, 0, InstancePaddingStrategy::Zero) } fn assign_instances( diff --git a/ceno_zkvm/src/tables/ram/ram_impl.rs b/ceno_zkvm/src/tables/ram/ram_impl.rs index 07f36cdab..98aa77ad7 100644 --- a/ceno_zkvm/src/tables/ram/ram_impl.rs +++ b/ceno_zkvm/src/tables/ram/ram_impl.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, mem::MaybeUninit}; +use std::marker::PhantomData; use ceno_emul::{Addr, Cycle}; use ff_ext::ExtensionField; @@ -10,7 +10,10 @@ use crate::{ circuit_builder::{CircuitBuilder, DynamicAddr, SetTableAddrType, SetTableSpec}, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, - instructions::riscv::constants::{LIMB_BITS, LIMB_MASK}, + instructions::{ + InstancePaddingStrategy, + riscv::constants::{LIMB_BITS, LIMB_MASK}, + }, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ProgramParams, @@ -116,7 +119,12 @@ impl NonVolatileTableConfig::new(NVRAM::len(&self.params), num_fixed); + dbg!(NVRAM::len(&self.params), num_fixed); + let mut init_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_fixed, + InstancePaddingStrategy::Zero, + ); assert_eq!(init_table.num_padding_instances(), 0); init_table @@ -146,7 +154,11 @@ impl NonVolatileTableConfig Result, ZKVMError> { - let mut final_table = RowMajorMatrix::::new(NVRAM::len(&self.params), num_witness); + let mut final_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_witness, + InstancePaddingStrategy::Zero, + ); final_table .par_iter_mut() @@ -246,7 +258,11 @@ impl PubIOTableConfig { ) -> RowMajorMatrix { assert!(NVRAM::len(&self.params).is_power_of_two()); - let mut init_table = RowMajorMatrix::::new(NVRAM::len(&self.params), num_fixed); + let mut init_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_fixed, + InstancePaddingStrategy::Zero, + ); assert_eq!(init_table.num_padding_instances(), 0); init_table @@ -265,7 +281,11 @@ impl PubIOTableConfig { num_witness: usize, final_cycles: &[Cycle], ) -> Result, ZKVMError> { - let mut final_table = RowMajorMatrix::::new(NVRAM::len(&self.params), num_witness); + let mut final_table = RowMajorMatrix::::new( + NVRAM::len(&self.params), + num_witness, + InstancePaddingStrategy::Zero, + ); final_table .par_iter_mut() @@ -369,8 +389,9 @@ impl DynVolatileRamTableConfig ) -> Result, ZKVMError> { assert!(final_mem.len() <= DVRAM::max_len(&self.params)); assert!(DVRAM::max_len(&self.params).is_power_of_two()); + dbg!(final_mem.len(), num_witness); let mut final_table = - RowMajorMatrix::::new(final_mem.len().next_power_of_two(), num_witness); + RowMajorMatrix::::new(final_mem.len(), num_witness, InstancePaddingStrategy::Zero); final_table .par_iter_mut() @@ -391,28 +412,6 @@ impl DynVolatileRamTableConfig set_val!(row, self.final_cycle, rec.cycle); }); - // set padding with well-form address - if final_mem.len().next_power_of_two() - final_mem.len() > 0 { - let paddin_entry_start = final_mem.len(); - final_table - .par_iter_mut() - .skip(final_mem.len()) - .enumerate() - .with_min_len(MIN_PAR_SIZE) - .for_each(|(i, row)| { - // Assign value limbs. - self.final_v.iter().for_each(|limb| { - set_val!(row, limb, 0u64); - }); - set_val!( - row, - self.addr, - DVRAM::addr(&self.params, paddin_entry_start + i) as u64 - ); - set_val!(row, self.final_cycle, 0_u64); - }); - } - Ok(final_table) } } diff --git a/ceno_zkvm/src/tables/range/range_circuit.rs b/ceno_zkvm/src/tables/range/range_circuit.rs index bb7c83448..7146270ce 100644 --- a/ceno_zkvm/src/tables/range/range_circuit.rs +++ b/ceno_zkvm/src/tables/range/range_circuit.rs @@ -44,8 +44,8 @@ impl TableCircuit for RangeTableCircuit num_fixed: usize, _input: &(), ) -> RowMajorMatrix { - let mut table = config.generate_fixed_traces(num_fixed, RANGE::content()); - Self::padding_zero(&mut table, num_fixed).expect("padding error"); + let table = config.generate_fixed_traces(num_fixed, RANGE::content()); + // Self::padding_zero(&mut table, num_fixed).expect("padding error"); table } @@ -56,8 +56,8 @@ impl TableCircuit for RangeTableCircuit _input: &(), ) -> Result, ZKVMError> { let multiplicity = &multiplicity[RANGE::ROM_TYPE as usize]; - let mut table = config.assign_instances(num_witin, multiplicity, RANGE::len())?; - Self::padding_zero(&mut table, num_witin).expect("padding error"); + let table = config.assign_instances(num_witin, multiplicity, RANGE::len())?; + // Self::padding_zero(&mut table, num_witin).expect("padding error"); Ok(table) } } diff --git a/ceno_zkvm/src/tables/range/range_impl.rs b/ceno_zkvm/src/tables/range/range_impl.rs index 505ef4b4f..aefbef232 100644 --- a/ceno_zkvm/src/tables/range/range_impl.rs +++ b/ceno_zkvm/src/tables/range/range_impl.rs @@ -3,12 +3,13 @@ use ff_ext::ExtensionField; use goldilocks::SmallField; use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; -use std::{collections::HashMap, mem::MaybeUninit}; +use std::collections::HashMap; use crate::{ circuit_builder::CircuitBuilder, error::ZKVMError, expression::{Expression, Fixed, ToExpr, WitIn}, + instructions::InstancePaddingStrategy, scheme::constants::MIN_PAR_SIZE, set_fixed_val, set_val, structs::ROMType, @@ -42,7 +43,9 @@ impl RangeTableConfig { num_fixed: usize, content: Vec, ) -> RowMajorMatrix { - let mut fixed = RowMajorMatrix::::new(content.len(), num_fixed); + dbg!(content.len(), num_fixed); + let mut fixed = + RowMajorMatrix::::new(content.len(), num_fixed, InstancePaddingStrategy::Zero); fixed .par_iter_mut() @@ -61,7 +64,9 @@ impl RangeTableConfig { multiplicity: &HashMap, length: usize, ) -> Result, ZKVMError> { - let mut witness = RowMajorMatrix::::new(length, num_witin); + dbg!(length, num_witin); + let mut witness = + RowMajorMatrix::::new(length, num_witin, InstancePaddingStrategy::Zero); let mut mlts = vec![0; length]; for (idx, mlt) in multiplicity { diff --git a/ceno_zkvm/src/uint.rs b/ceno_zkvm/src/uint.rs index afb231d8a..1f540b472 100644 --- a/ceno_zkvm/src/uint.rs +++ b/ceno_zkvm/src/uint.rs @@ -20,7 +20,7 @@ use goldilocks::SmallField; use itertools::Itertools; use std::{ borrow::Cow, - mem::{self, MaybeUninit}, + mem::{self}, ops::Index, }; pub use strum::IntoEnumIterator; @@ -181,20 +181,20 @@ impl UIntLimbs { pub fn assign_value + Default + From + Copy>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], value: Value, ) { self.assign_limbs(instance, value.as_u16_limbs()) } - pub fn assign_add_outcome(&self, instance: &mut [MaybeUninit], value: &ValueAdd) { + pub fn assign_add_outcome(&self, instance: &mut [E::BaseField], value: &ValueAdd) { self.assign_limbs(instance, &value.limbs); self.assign_carries(instance, &value.carries); } pub fn assign_mul_outcome( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, value: &ValueMul, ) -> Result<(), ZKVMError> { @@ -203,7 +203,7 @@ impl UIntLimbs { self.assign_carries_auxiliary(instance, lkm, &value.carries, value.max_carry_value) } - pub fn assign_limbs(&self, instance: &mut [MaybeUninit], limbs_values: &[u16]) { + pub fn assign_limbs(&self, instance: &mut [E::BaseField], limbs_values: &[u16]) { assert!( limbs_values.len() <= Self::NUM_LIMBS, "assign input length mismatch. input_len={}, NUM_CELLS={}", @@ -217,14 +217,14 @@ impl UIntLimbs { .map(|v| E::BaseField::from(*v as u64)) .chain(std::iter::repeat(E::BaseField::ZERO)), ) { - instance[wire.id as usize] = MaybeUninit::new(limb); + instance[wire.id as usize] = limb; } } } pub fn assign_carries + Copy>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], carry_values: &[T], ) { assert!( @@ -243,14 +243,14 @@ impl UIntLimbs { .map(|v| E::BaseField::from(Into::::into(*v))) .chain(std::iter::repeat(E::BaseField::ZERO)), ) { - instance[wire.id as usize] = MaybeUninit::new(carry); + instance[wire.id as usize] = carry; } } } pub fn assign_carries_auxiliary + Copy>( &self, - instance: &mut [MaybeUninit], + instance: &mut [E::BaseField], lkm: &mut LkMultiplicity, carry_values: &[T], max_carry: u64, diff --git a/ceno_zkvm/src/witness.rs b/ceno_zkvm/src/witness.rs index e85360aee..3e39fbd90 100644 --- a/ceno_zkvm/src/witness.rs +++ b/ceno_zkvm/src/witness.rs @@ -1,25 +1,26 @@ use ff::Field; +use itertools::Itertools; use std::{ array, cell::RefCell, collections::HashMap, - mem::{self, MaybeUninit}, + iter, + mem::{self}, ops::Index, slice::{Chunks, ChunksMut}, sync::Arc, + time::Instant, }; -use multilinear_extensions::{ - mle::{DenseMultilinearExtension, IntoMLEs}, - util::create_uninit_vec, -}; +use multilinear_extensions::mle::{DenseMultilinearExtension, IntoMLE}; use rayon::{ - iter::{IndexedParallelIterator, IntoParallelRefMutIterator, ParallelIterator}, + iter::{IntoParallelRefIterator, ParallelIterator}, slice::ParallelSliceMut, }; use thread_local::ThreadLocal; use crate::{ + instructions::InstancePaddingStrategy, structs::ROMType, tables::{AndTable, LtuTable, OpsTable, OrTable, PowTable, XorTable}, utils::next_pow2_instance_padding, @@ -28,98 +29,107 @@ use crate::{ #[macro_export] macro_rules! set_val { ($ins:ident, $field:expr, $val:expr) => { - $ins[$field.id as usize] = MaybeUninit::new($val.into()); + $ins[$field.id as usize] = $val.into(); }; } #[macro_export] macro_rules! set_fixed_val { ($ins:ident, $field:expr, $val:expr) => { - $ins[$field.0] = MaybeUninit::new($val); + $ins[$field.0] = $val; }; } #[derive(Clone)] pub struct RowMajorMatrix { // represent 2D in 1D linear memory and avoid double indirection by Vec> to improve performance - values: Vec>, - num_padding_rows: usize, + values: Vec, num_col: usize, + padding_strategy: InstancePaddingStrategy, } -impl RowMajorMatrix { - pub fn new(num_rows: usize, num_col: usize) -> Self { - let num_total_rows = next_pow2_instance_padding(num_rows); - let num_padding_rows = num_total_rows - num_rows; +impl RowMajorMatrix { + pub fn new(num_rows: usize, num_col: usize, padding_strategy: InstancePaddingStrategy) -> Self { + // assert!(false); RowMajorMatrix { - values: create_uninit_vec(num_total_rows * num_col), - num_padding_rows, + values: vec![T::default(); num_rows * num_col], num_col, + padding_strategy, } } - pub fn num_instances(&self) -> usize { - self.values.len() / self.num_col - self.num_padding_rows + pub fn len(&self) -> usize { + self.values.len() + } + + pub fn values(&mut self) -> &mut Vec { + &mut self.values } pub fn num_padding_instances(&self) -> usize { - self.num_padding_rows + return next_pow2_instance_padding(self.num_instances()) - self.num_instances(); + } + + pub fn num_instances(&self) -> usize { + self.values.len() / self.num_col } - pub fn iter_rows(&self) -> Chunks> { + pub fn iter_rows(&self) -> Chunks { self.values.chunks(self.num_col) } - pub fn iter_mut(&mut self) -> ChunksMut> { + pub fn iter_mut(&mut self) -> ChunksMut { self.values.chunks_mut(self.num_col) } - pub fn par_iter_mut(&mut self) -> rayon::slice::ChunksMut> { + pub fn par_iter_mut(&mut self) -> rayon::slice::ChunksMut { self.values.par_chunks_mut(self.num_col) } - pub fn par_batch_iter_mut( - &mut self, - num_rows: usize, - ) -> rayon::slice::ChunksMut> { + pub fn par_batch_iter_mut(&mut self, num_rows: usize) -> rayon::slice::ChunksMut { self.values.par_chunks_mut(num_rows * self.num_col) } - - pub fn par_batch_iter_padding_mut( - &mut self, - num_instances: Option, - batch_size: usize, - ) -> rayon::slice::ChunksMut<'_, MaybeUninit> { - let num_instances = num_instances.unwrap_or(self.num_instances()); - self.values[num_instances * self.num_col..] - .as_mut() - .par_chunks_mut(batch_size * self.num_col) - } - - pub fn de_interleaving(mut self) -> Vec> { - (0..self.num_col) - .map(|i| { - self.values - .par_iter_mut() - .skip(i) - .step_by(self.num_col) - .map(|v| unsafe { mem::replace(v, mem::MaybeUninit::uninit()).assume_init() }) - .collect::>() - }) - .collect() - } } impl RowMajorMatrix { pub fn into_mles>( self, ) -> Vec> { - self.de_interleaving().into_mles() + let start = Instant::now(); + let padding_row = match self.padding_strategy { + // If asked to repeat and actually have content to repeat + InstancePaddingStrategy::RepeatLast if self.values.len() > 0 => { + self.values[self.values.len() - self.num_col..].to_vec() + } + // Otherwise zeros + _ => vec![F::ZERO; self.num_col], + }; + let num_padding = self.num_padding_instances(); + let result = (0..self.num_col) + .collect_vec() + .par_iter() + .map(|i| { + self.values + .iter() + .skip(*i) + .step_by(self.num_col) + .chain(&mut iter::repeat(&padding_row[*i]).take(num_padding)) + .map(|val| *val) + .collect::>() + .into_mle() + }) + .collect(); + let size = self.num_col * self.len(); + if size > 1000 * 1000 { + let duration = start.elapsed().as_secs_f64(); + println!("Time taken: {:?}, size: {:?}", duration, size); + } + result } } impl Index for RowMajorMatrix { - type Output = [MaybeUninit]; + type Output = [F]; fn index(&self, idx: usize) -> &Self::Output { &self.values[self.num_col * idx..][..self.num_col] diff --git a/multilinear_extensions/src/mle.rs b/multilinear_extensions/src/mle.rs index 8a182e645..dc7b38d9c 100644 --- a/multilinear_extensions/src/mle.rs +++ b/multilinear_extensions/src/mle.rs @@ -103,9 +103,10 @@ pub trait IntoMLE: Sized { } impl IntoMLE> for Vec { - fn into_mle(mut self) -> DenseMultilinearExtension { + fn into_mle(self) -> DenseMultilinearExtension { let next_pow2 = self.len().next_power_of_two(); - self.resize(next_pow2, F::ZERO); + // self.resize(next_pow2, F::ZERO); + assert!(self.len().is_power_of_two(), "{}", self.len()); DenseMultilinearExtension::from_evaluation_vec_smart::(ceil_log2(next_pow2), self) } } @@ -118,6 +119,8 @@ impl> IntoMLEs> { fn into_mles(self) -> Vec> { + // TODO: figure out correct place to test + // assert!(self.len().is_power_of_two(), "{}", self.len()); self.into_iter().map(|v| v.into_mle()).collect() } } diff --git a/multilinear_extensions/src/virtual_poly.rs b/multilinear_extensions/src/virtual_poly.rs index 72c5b2649..094d640db 100644 --- a/multilinear_extensions/src/virtual_poly.rs +++ b/multilinear_extensions/src/virtual_poly.rs @@ -313,7 +313,7 @@ impl VirtualPolynomial { let mut flattened_ml_extensions = vec![]; let mut hm = HashMap::new(); for mle in self.flattened_ml_extensions.iter() { - let mle_ptr = Arc::as_ptr(mle) as usize; + let mle_ptr = Arc::as_ptr(&mle) as usize; let index = self.raw_pointers_lookup_table.get(&mle_ptr).unwrap(); let mle_ext_field = mle.as_ref().to_ext_field();