Skip to content

Commit

Permalink
Use proper self for BrotliEncoderStateStruct
Browse files Browse the repository at this point in the history 
Big refactoring, updating all BrotliEncoderStateStruct-using functions,
renaming them, and converting many to return bool.
  • Loading branch information
@nyurik @danielrh
nyurik authored and danielrh committed Apr 7, 2024
1 parent 03d5501 commit 2ff7b21
Show file tree
Hide file tree
Showing 8 changed files with 1,203 additions and 1,205 deletions.
2,200 changes: 1,110 additions & 1,090 deletions src/enc/encode.rs
View file

Large diffs are not rendered by default.

22 changes: 9 additions & 13 deletions src/enc/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,9 @@ mod parameters;
mod test;
mod weights;
pub use self::backward_references::{BrotliEncoderParams, UnionHasher};
use self::encode::{
BrotliEncoderCompressStream, BrotliEncoderCreateInstance, BrotliEncoderDestroyInstance,
BrotliEncoderIsFinished, BrotliEncoderOperation, BrotliEncoderSetCustomDictionary,
};
use self::encode::{BrotliEncoderDestroyInstance, BrotliEncoderOperation};
pub use self::encode::{
BrotliEncoderInitParams, BrotliEncoderMaxCompressedSize, BrotliEncoderMaxCompressedSizeMulti,
BrotliEncoderSetParameter,
};
pub use self::hash_to_binary_tree::ZopfliNode;
pub use self::interface::StaticCommand;
Expand All @@ -87,6 +83,7 @@ use std::io::{Error, ErrorKind, Read, Write};

#[cfg(feature = "std")]
pub use brotli_decompressor::{IntoIoReader, IoReaderWrapper, IoWriterWrapper};
use enc::encode::BrotliEncoderStateStruct;

#[cfg(not(feature = "std"))]
pub use self::singlethreading::{compress_worker_pool, new_work_pool, WorkerPool};
Expand Down Expand Up @@ -258,10 +255,10 @@ where
{
assert!(!input_buffer.is_empty());
assert!(!output_buffer.is_empty());
let mut s_orig = BrotliEncoderCreateInstance(alloc);
let mut s_orig = BrotliEncoderStateStruct::new(alloc);
s_orig.params = params.clone();
if !dict.is_empty() {
BrotliEncoderSetCustomDictionary(&mut s_orig, dict.len(), dict);
s_orig.set_custom_dictionary(dict.len(), dict);
}
let mut next_in_offset: usize = 0;
let mut next_out_offset: usize = 0;
Expand Down Expand Up @@ -300,8 +297,7 @@ where
} else {
op = BrotliEncoderOperation::BROTLI_OPERATION_PROCESS;
}
let result = BrotliEncoderCompressStream(
s,
let result = s.compress_stream(
op,
&mut available_in,
input_buffer,
Expand All @@ -312,8 +308,8 @@ where
&mut total_out,
metablock_callback,
);
let fin = BrotliEncoderIsFinished(s);
if available_out == 0 || fin != 0 {
let fin = s.is_finished();
if available_out == 0 || fin {
let lim = output_buffer.len() - available_out;
assert_eq!(next_out_offset, lim);
next_out_offset = 0;
Expand All @@ -332,13 +328,13 @@ where
available_out = output_buffer.len();
next_out_offset = 0;
}
if result <= 0 {
if !result {
if read_err.is_ok() {
read_err = Err(unexpected_eof_error_constant);
}
break;
}
if fin != 0 {
if fin {
break;
}
}
Expand Down
31 changes: 12 additions & 19 deletions src/enc/reader.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
use super::backward_references::BrotliEncoderParams;
use super::combined_alloc::BrotliAlloc;
use super::encode::{
BrotliEncoderCompressStream, BrotliEncoderCreateInstance, BrotliEncoderDestroyInstance,
BrotliEncoderIsFinished, BrotliEncoderOperation, BrotliEncoderParameter,
BrotliEncoderSetParameter, BrotliEncoderStateStruct,
BrotliEncoderDestroyInstance, BrotliEncoderOperation, BrotliEncoderParameter,
BrotliEncoderStateStruct,
};
use super::interface;
use brotli_decompressor::CustomRead;
Expand Down Expand Up @@ -146,19 +145,15 @@ impl<ErrType, R: CustomRead<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: Br
input_len: 0,
input_eof: false,
input: r,
state: StateWrapper(BrotliEncoderCreateInstance(alloc)),
state: StateWrapper(BrotliEncoderStateStruct::new(alloc)),
error_if_invalid_data: Some(invalid_data_error_type),
};
BrotliEncoderSetParameter(
&mut ret.state.0,
BrotliEncoderParameter::BROTLI_PARAM_QUALITY,
q,
);
BrotliEncoderSetParameter(
&mut ret.state.0,
BrotliEncoderParameter::BROTLI_PARAM_LGWIN,
lgwin,
);
ret.state
.0
.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_QUALITY, q);
ret.state
.0
.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_LGWIN, lgwin);

ret
}
Expand Down Expand Up @@ -232,8 +227,7 @@ impl<ErrType, R: CustomRead<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: Br
} else {
op = BrotliEncoderOperation::BROTLI_OPERATION_PROCESS;
}
let ret = BrotliEncoderCompressStream(
&mut self.state.0,
let ret = self.state.0.compress_stream(
op,
&mut avail_in,
self.input_buffer.slice_mut(),
Expand All @@ -247,11 +241,10 @@ impl<ErrType, R: CustomRead<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: Br
if avail_in == 0 {
self.copy_to_front();
}
if ret <= 0 {
if !ret {
return Err(self.error_if_invalid_data.take().unwrap());
}
let fin = BrotliEncoderIsFinished(&mut self.state.0);
if fin != 0 {
if self.state.0.is_finished() {
break;
}
}
Expand Down
38 changes: 15 additions & 23 deletions src/enc/test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,7 @@ use super::super::alloc::{
bzero, AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator,
};
use super::cluster::HistogramPair;
use super::encode::{
BrotliEncoderCompressStream, BrotliEncoderCreateInstance, BrotliEncoderDestroyInstance,
BrotliEncoderIsFinished, BrotliEncoderOperation, BrotliEncoderParameter,
BrotliEncoderSetParameter,
};
use super::encode::{BrotliEncoderOperation, BrotliEncoderParameter};
use super::histogram::{ContextType, HistogramCommand, HistogramDistance, HistogramLiteral};
use super::StaticCommand;
use super::ZopfliNode;
Expand All @@ -31,6 +27,7 @@ use super::interface;
use super::pdf::PDF;
use brotli_decompressor::HuffmanCode;
use core::ops;
use enc::encode::BrotliEncoderStateStruct;

declare_stack_allocator_struct!(MemPool, 128, stack);
declare_stack_allocator_struct!(CallocatedFreelist4096, 128, calloc);
Expand All @@ -46,7 +43,7 @@ fn oneshot_compress(
magic: bool,
in_batch_size: usize,
out_batch_size: usize,
) -> (i32, usize) {
) -> (bool, usize) {
let stack_u8_buffer =
unsafe { define_allocator_memory_pool!(96, u8, [0; 24 * 1024 * 1024], calloc) };
let stack_u16_buffer =
Expand Down Expand Up @@ -113,7 +110,7 @@ fn oneshot_compress(
let mhp = CallocatedFreelist2048::<HistogramPair>::new_allocator(stack_hp_buffer.data, bzero);
let mct = CallocatedFreelist2048::<ContextType>::new_allocator(stack_ct_buffer.data, bzero);
let mht = CallocatedFreelist2048::<HuffmanTree>::new_allocator(stack_ht_buffer.data, bzero);
let mut s_orig = BrotliEncoderCreateInstance(CombiningAllocator::new(
let mut s_orig = BrotliEncoderStateStruct::new(CombiningAllocator::new(
stack_u8_allocator,
stack_u16_allocator,
stack_i32_allocator,
Expand All @@ -138,21 +135,19 @@ fn oneshot_compress(
{
let s = &mut s_orig;

BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_QUALITY, quality);
s.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_QUALITY, quality);
if magic {
BrotliEncoderSetParameter(
s,
s.set_parameter(
BrotliEncoderParameter::BROTLI_PARAM_MAGIC_NUMBER,
magic as u32,
);
}
if quality >= 10 {
BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_Q9_5, 1);
s.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_Q9_5, 1);
}
BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_LGWIN, lgwin);
BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_MODE, 0); // gen, text, font
BrotliEncoderSetParameter(
s,
s.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_LGWIN, lgwin);
s.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_MODE, 0); // gen, text, font
s.set_parameter(
BrotliEncoderParameter::BROTLI_PARAM_SIZE_HINT,
input.len() as u32,
);
Expand Down Expand Up @@ -194,8 +189,7 @@ fn oneshot_compress(
_,
>| ();

let result = BrotliEncoderCompressStream(
s,
let result = s.compress_stream(
op,
&mut available_in,
input,
Expand All @@ -206,18 +200,16 @@ fn oneshot_compress(
&mut total_out,
&mut nop_callback,
);
if result <= 0 {
if !result {
return (result, next_out_offset);
}
if BrotliEncoderIsFinished(s) != 0 {
if s.is_finished() {
break;
}
}

BrotliEncoderDestroyInstance(s);
}

(1, next_out_offset)
(true, next_out_offset)
}

fn oneshot_decompress(compressed: &[u8], output: &mut [u8]) -> (BrotliResult, usize, usize) {
Expand Down Expand Up @@ -272,7 +264,7 @@ fn oneshot(
in_buffer_size,
out_buffer_size,
);
if success == 0 {
if !success {
//return (BrotliResult::ResultFailure, 0, 0);
available_in = compressed.len();
}
Expand Down
17 changes: 8 additions & 9 deletions src/enc/threading.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use super::backward_references::{AnyHasher, BrotliEncoderParams, CloneWithAlloc, UnionHasher};
use super::encode::{
BrotliEncoderCompressStream, BrotliEncoderCreateInstance, BrotliEncoderDestroyInstance,
BrotliEncoderMaxCompressedSize, BrotliEncoderOperation,
BrotliEncoderSetCustomDictionaryWithOptionalPrecomputedHasher, HasherSetup, SanitizeParams,
BrotliEncoderDestroyInstance, BrotliEncoderMaxCompressedSize, BrotliEncoderOperation,
HasherSetup, SanitizeParams,
};
use super::BrotliAlloc;
use alloc::{Allocator, SliceWrapper, SliceWrapperMut};
Expand All @@ -11,8 +10,10 @@ use core::any;
use core::marker::PhantomData;
use core::mem;
use core::ops::Range;
use enc::encode::BrotliEncoderStateStruct;
#[cfg(feature = "std")]
use std;

pub type PoisonedThreadError = ();

#[cfg(feature = "std")]
Expand Down Expand Up @@ -338,16 +339,15 @@ where
&mut alloc,
BrotliEncoderMaxCompressedSize(range.end - range.start),
);
let mut state = BrotliEncoderCreateInstance(alloc);
let mut state = BrotliEncoderStateStruct::new(alloc);
state.params = input_and_params.1.clone();
if thread_index != 0 {
state.params.catable = true; // make sure we can concatenate this to the other work results
state.params.magic_number = false; // no reason to pepper this around
}
state.params.appendable = true; // make sure we are at least appendable, so that future items can be catted in
if thread_index != 0 {
BrotliEncoderSetCustomDictionaryWithOptionalPrecomputedHasher(
&mut state,
state.set_custom_dictionary_with_optional_precomputed_hasher(
range.start,
&input_and_params.0.slice()[..range.start],
hasher,
Expand All @@ -359,8 +359,7 @@ where
loop {
let mut next_in_offset = 0usize;
let mut available_in = range.end - range.start;
let result = BrotliEncoderCompressStream(
&mut state,
let result = state.compress_stream(
BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
&mut available_in,
&input_and_params.0.slice()[range.clone()],
Expand All @@ -373,7 +372,7 @@ where
);
let new_range = range.start + next_in_offset..range.end;
range = new_range;
if result != 0 {
if result {
compression_result = Ok(out_offset);
break;
} else if available_out == 0 {
Expand Down
35 changes: 13 additions & 22 deletions src/enc/writer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@
use super::backward_references::BrotliEncoderParams;
use super::combined_alloc::BrotliAlloc;
use super::encode::{
BrotliEncoderCompressStream, BrotliEncoderCreateInstance, BrotliEncoderDestroyInstance,
BrotliEncoderHasMoreOutput, BrotliEncoderIsFinished, BrotliEncoderOperation,
BrotliEncoderParameter, BrotliEncoderSetParameter, BrotliEncoderStateStruct,
BrotliEncoderDestroyInstance, BrotliEncoderOperation, BrotliEncoderParameter,
BrotliEncoderStateStruct,
};
use super::interface;
pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
Expand Down Expand Up @@ -156,19 +155,13 @@ impl<ErrType, W: CustomWrite<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: B
output_buffer: buffer,
total_out: Some(0),
output: Some(w),
state: BrotliEncoderCreateInstance(alloc),
state: BrotliEncoderStateStruct::new(alloc),
error_if_invalid_data: Some(invalid_data_error_type),
};
BrotliEncoderSetParameter(
&mut ret.state,
BrotliEncoderParameter::BROTLI_PARAM_QUALITY,
q,
);
BrotliEncoderSetParameter(
&mut ret.state,
BrotliEncoderParameter::BROTLI_PARAM_LGWIN,
lgwin,
);
ret.state
.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_QUALITY, q);
ret.state
.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_LGWIN, lgwin);

ret
}
Expand All @@ -184,8 +177,7 @@ impl<ErrType, W: CustomWrite<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: B
let mut input_offset: usize = 0;
let mut avail_out: usize = self.output_buffer.slice_mut().len();
let mut output_offset: usize = 0;
let ret = BrotliEncoderCompressStream(
&mut self.state,
let ret = self.state.compress_stream(
op,
&mut avail_in,
&[],
Expand All @@ -205,16 +197,16 @@ impl<ErrType, W: CustomWrite<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: B
Err(e) => return Err(e),
}
}
if ret <= 0 {
if !ret {
return Err(self.error_if_invalid_data.take().unwrap());
}
if let BrotliEncoderOperation::BROTLI_OPERATION_FLUSH = op {
if BrotliEncoderHasMoreOutput(&mut self.state) != 0 {
if self.state.has_more_output() {
continue;
}
return Ok(());
}
if BrotliEncoderIsFinished(&mut self.state) != 0 {
if self.state.is_finished() {
return Ok(());
}
}
Expand Down Expand Up @@ -262,8 +254,7 @@ impl<ErrType, W: CustomWrite<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: B
while avail_in != 0 {
let mut output_offset = 0;
let mut avail_out = self.output_buffer.slice_mut().len();
let ret = BrotliEncoderCompressStream(
&mut self.state,
let ret = self.state.compress_stream(
BrotliEncoderOperation::BROTLI_OPERATION_PROCESS,
&mut avail_in,
buf,
Expand All @@ -283,7 +274,7 @@ impl<ErrType, W: CustomWrite<ErrType>, BufferType: SliceWrapperMut<u8>, Alloc: B
Err(e) => return Err(e),
}
}
if ret <= 0 {
if !ret {
return Err(self.error_if_invalid_data.take().unwrap());
}
}
Expand Down
Loading

0 comments on commit 2ff7b21

Please sign in to comment.