Skip to content

Commit

Permalink
Merge branch 'master' into huffman
Browse files Browse the repository at this point in the history
  • Loading branch information
@ayzk
ayzk committed Nov 1, 2023
2 parents ebe3639 + d2a03ea commit 4113ad1
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 59 deletions.
14 changes: 7 additions & 7 deletions include/SZ3/api/impl/SZLorenzoReg.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ namespace SZ3 {
if (use_single_predictor) {
return make_sz_general_compressor<T, N>(
make_sz_general_frontend<T, N>(conf, RegressionPredictor<T, N>(conf.blockSize, conf.absErrorBound),
quantizer), encoder, lossless);
quantizer), encoder, lossless);
} else {
predictors.push_back(std::make_shared<RegressionPredictor<T, N>>(conf.blockSize, conf.absErrorBound));
}
Expand All @@ -63,14 +63,14 @@ namespace SZ3 {
if (use_single_predictor) {
return make_sz_general_compressor<T, N>(
make_sz_general_frontend<T, N>(conf, PolyRegressionPredictor<T, N>(conf.blockSize, conf.absErrorBound),
quantizer), encoder, lossless);
quantizer), encoder, lossless);
} else {
predictors.push_back(std::make_shared<PolyRegressionPredictor<T, N>>(conf.blockSize, conf.absErrorBound));
}
}
return make_sz_general_compressor<T, N>(
make_sz_general_frontend<T, N>(conf, ComposedPredictor<T, N>(predictors),
quantizer), encoder, lossless);
quantizer), encoder, lossless);
}


Expand All @@ -83,10 +83,10 @@ namespace SZ3 {

char *cmpData;
auto quantizer = LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2);
if (N == 3 && !conf.regression2) {
if (N == 3 && !conf.regression2 || (N == 1 && !conf.regression && !conf.regression2)) {
// use fast version for 3D
auto sz = make_sz_general_compressor<T, N>(make_sz_fast_frontend<T, N>(conf, quantizer), HuffmanEncoder<int>(),
Lossless_zstd());
Lossless_zstd());
cmpData = (char *) sz->compress(conf, data, outSize);
} else {
auto sz = make_lorenzo_regression_compressor<T, N>(conf, quantizer, HuffmanEncoder<int>(), Lossless_zstd());
Expand All @@ -102,10 +102,10 @@ namespace SZ3 {

uchar const *cmpDataPos = (uchar *) cmpData;
LinearQuantizer<T> quantizer;
if (N == 3 && !conf.regression2) {
if (N == 3 && !conf.regression2 || (N == 1 && !conf.regression && !conf.regression2)) {
// use fast version for 3D
auto sz = make_sz_general_compressor<T, N>(make_sz_fast_frontend<T, N>(conf, quantizer),
HuffmanEncoder<int>(), Lossless_zstd());
HuffmanEncoder<int>(), Lossless_zstd());
sz->decompress(cmpDataPos, cmpSize, decData);
return;

Expand Down
120 changes: 68 additions & 52 deletions include/SZ3/frontend/SZFastFrontend.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ namespace SZ3 {
conf.regression, conf.absErrorBound),
precision(conf.absErrorBound),
conf(conf) {
assert(N == 3 && "SZMeta Front only support 3D data");
assert((N == 1 || N == 3) && "SZMeta Front only support 1D or 3D data");
}

~SZFastFrontend() {
Expand All @@ -39,22 +39,30 @@ namespace SZ3 {


std::vector<int> compress(T *data) {
return compress_3d(data);
if (N == 1) {
return compress_1d(data);
} else if (N == 3) {
return compress_3d(data);
}
};

T *decompress(std::vector<int> &quant_inds, T *dec_data) {
return decompress_3d(quant_inds, dec_data);
if (N == 1) {
return decompress_1d(quant_inds, dec_data);
} else {
return decompress_3d(quant_inds, dec_data);
}
};


void save(uchar *&c) {

write(params, c);
write(precision, c);
if (N == 3) {
write(params, c);
write(precision, c);
// write(intv_radius, c);
write(mean_info.use_mean, c);
write(mean_info.mean, c);
write(reg_count, c);
write(mean_info.use_mean, c);
write(mean_info.mean, c);
write(reg_count, c);
// write(unpred_count_buffer, size.block_size * size.block_size, c);
// T *unpred_data_buffer_pos = unpred_data_buffer;
// for (int i = 0; i < size.block_size; i++) {
Expand All @@ -69,16 +77,17 @@ namespace SZ3 {

// Huffman_encode_tree_and_data(SELECTOR_RADIUS, indicator, size.num_blocks, c);
// indicator_huffman.preprocess_encode(indicator, SELECTOR_RADIUS);
indicator_huffman.save(c);
indicator_huffman.encode(indicator, c);
indicator_huffman.postprocess_encode();
auto *c2 = c;
indicator_huffman.save(c);
indicator_huffman.encode(indicator, c);
indicator_huffman.postprocess_encode();
auto *c2 = c;

// convertIntArray2ByteArray_fast_1b_to_result_sz(indicator, size.num_blocks, c);

if (reg_count) {
encode_regression_coefficients(reg_params_type, reg_unpredictable_data, RegCoeffNum3d * reg_count,
reg_unpredictable_data_pos - reg_unpredictable_data, reg_huffman, c);
if (reg_count) {
encode_regression_coefficients(reg_params_type, reg_unpredictable_data, RegCoeffNum3d * reg_count,
reg_unpredictable_data_pos - reg_unpredictable_data, reg_huffman, c);
}
}

quantizer.save(c);
Expand All @@ -87,48 +96,29 @@ namespace SZ3 {
void load(const uchar *&c, size_t &remaining_length) {
clear();
const uchar *c_pos = c;
if (N == 3) {

read(params, c, remaining_length);
read(precision, c, remaining_length);
read(params, c, remaining_length);
read(precision, c, remaining_length);
// read(intv_radius, c, remaining_length);
read(mean_info.use_mean, c, remaining_length);
read(mean_info.mean, c, remaining_length);
read(reg_count, c, remaining_length);
read(mean_info.use_mean, c, remaining_length);
read(mean_info.mean, c, remaining_length);
read(reg_count, c, remaining_length);

size_t r1 = conf.dims[0];
size_t r2 = conf.dims[1];
size_t r3 = conf.dims[2];
size = SZMETA::DSize_3d(r1, r2, r3, params.block_size);
// prepare unpred buffer for vectorization
est_unpred_count_per_index = size.num_blocks * size.block_size * 1;
// if(!params.block_independant) est_unpred_count_per_index /= 20;
// unpred_count_buffer = (int *) malloc(size.block_size * size.block_size * sizeof(T));
// read(unpred_count_buffer, size.block_size * size.block_size, c, remaining_length);
// unpred_data_buffer = (T *) malloc(
// size.block_size * size.block_size * est_unpred_count_per_index * sizeof(T));
// T *unpred_data_buffer_pos = unpred_data_buffer;
// for (int i = 0; i < size.block_size; i++) {
// for (int j = 0; j < size.block_size; j++) {
// memcpy(unpred_data_buffer_pos, c,
// unpred_count_buffer[i * size.block_size + j] * sizeof(T));
// c += unpred_count_buffer[i * size.block_size + j] * sizeof(T);
// unpred_data_buffer_pos += est_unpred_count_per_index;
// }
// }
// memset(unpred_count_buffer, 0, size.block_size * size.block_size * sizeof(int));
// unsigned char * indicator = convertByteArray2IntArray_fast_1b_sz(size.num_blocks, c, (size.num_blocks - 1)/8 + 1);
indicator_huffman = HuffmanEncoder<int>();
indicator_huffman.load(c, remaining_length);
indicator = indicator_huffman.decode(c, size.num_blocks);
indicator_huffman.postprocess_decode();
indicator_huffman = HuffmanEncoder<int>();
indicator_huffman.load(c, remaining_length);
indicator = indicator_huffman.decode(c, size.num_blocks);
indicator_huffman.postprocess_decode();


if (reg_count) {
reg_params = decode_regression_coefficients(c, reg_count, size.block_size, precision,
params);
if (reg_count) {
reg_params = decode_regression_coefficients(c, reg_count, size.block_size, precision,
params);
}
}
quantizer.load(c, remaining_length);
remaining_length -= c_pos - c;

}


Expand Down Expand Up @@ -168,9 +158,28 @@ namespace SZ3 {
return quantizer.get_radius();
}

size_t get_num_elements() const { return size.num_elements; };
size_t get_num_elements() const {
return conf.num;
};

private:
std::vector<int> compress_1d(T *data) {
std::vector<int> quant_bins(conf.num);
quant_bins[0] = quantizer.quantize_and_overwrite(data[0], 0);
for (size_t i = 1; i < conf.num; i++) {
quant_bins[i] = quantizer.quantize_and_overwrite(data[i], data[i - 1]);
}
return quant_bins;
}

T *decompress_1d(std::vector<int> &quant_inds, T *dec_data) {
dec_data[0] = quantizer.recover(0, quant_inds[0]);
for (size_t i = 1; i < conf.num; i++) {
dec_data[i] = quantizer.recover(dec_data[i - 1], quant_inds[i]);
}
return dec_data;
}

// unsigned char *
// compress_3d(const T *data, size_t r1, size_t r2, size_t r3, double precision, size_t &compressed_size,
// const SZMETA::meta_params &params, SZMETA::CompressStats &compress_info) {
Expand Down Expand Up @@ -374,7 +383,7 @@ namespace SZ3 {

if (reg_count) {
reg_huffman = HuffmanEncoder<int>();
reg_huffman.preprocess_encode(reg_params_type, RegCoeffNum3d * reg_count, RegCoeffRadius * 2);
reg_huffman.preprocess_encode(reg_params_type, RegCoeffNum3d * reg_count, 0);
}
indicator_huffman = HuffmanEncoder<int>();
indicator_huffman.preprocess_encode(indicator, SELECTOR_RADIUS);
Expand All @@ -388,6 +397,13 @@ namespace SZ3 {
// meta_decompress_3d(const unsigned char *compressed, size_t r1, size_t r2, size_t r3) {
T *decompress_3d(std::vector<int> &quant_inds, T *dec_data) {

size_t r1 = conf.dims[0];
size_t r2 = conf.dims[1];
size_t r3 = conf.dims[2];
size = SZMETA::DSize_3d(r1, r2, r3, params.block_size);
// prepare unpred buffer for vectorization
est_unpred_count_per_index = size.num_blocks * size.block_size * 1;

int *type = quant_inds.data();
// T *dec_data = new T[size.num_elements];
// dec_data_sp_float = (float *) dec_data;
Expand Down

0 comments on commit 4113ad1

Please sign in to comment.