Skip to content

Commit

Permalink
1. update annotations
Browse files Browse the repository at this point in the history 
2. fix make warnings
  • Loading branch information
@ayzk
ayzk committed Sep 16, 2024
1 parent dbd73bb commit 876a3ca
Show file tree
Hide file tree
Showing 65 changed files with 7,320 additions and 7,745 deletions.
42 changes: 0 additions & 42 deletions include/SZ3/api/impl/SZAlgo.hpp
View file

This file was deleted.

299 changes: 147 additions & 152 deletions include/SZ3/api/impl/SZAlgoInterp.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,170 +1,165 @@
#ifndef SZ3_SZALGOINTERP_HPP
#define SZ3_SZALGOINTERP_HPP
#ifndef SZ3_SZALGO_INTERP_HPP
#define SZ3_SZALGO_INTERP_HPP

#include "SZ3/decomposition/InterpolationDecomposition.hpp"
#include "SZ3/api/impl/SZAlgoLorenzoReg.hpp"
#include "SZ3/compressor/specialized/SZBlockInterpolationCompressor.hpp"
#include "SZ3/quantizer/IntegerQuantizer.hpp"
#include "SZ3/decomposition/InterpolationDecomposition.hpp"
#include "SZ3/lossless/Lossless_zstd.hpp"
#include "SZ3/utils/Iterator.hpp"
#include "SZ3/utils/Statistic.hpp"
#include "SZ3/quantizer/LinearQuantizer.hpp"
#include "SZ3/utils/Config.hpp"
#include "SZ3/utils/Extraction.hpp"
#include "SZ3/utils/QuantOptimizatioin.hpp"
#include "SZ3/utils/Config.hpp"
#include "SZ3/api/impl/SZAlgoLorenzoReg.hpp"
#include <cmath>
#include <memory>
#include "SZ3/utils/Statistic.hpp"

namespace SZ3 {
template<class T, uint N>
size_t SZ_compress_Interp(Config &conf, T *data, uchar *cmpData, size_t cmpCap) {
assert(N == conf.N);
assert(conf.cmprAlgo == ALGO_INTERP);
calAbsErrorBound(conf, data);

auto sz = make_compressor_sz_generic<T, N>(
make_decomposition_interpolation<T, N>(conf,
LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
HuffmanEncoder<int>(),
Lossless_zstd());
return sz->compress(conf, data, cmpData, cmpCap);
// return cmpData;
}

template<class T, uint N>
void SZ_decompress_Interp(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) {
assert(conf.cmprAlgo == ALGO_INTERP);
auto cmpDataPos = cmpData;
auto sz = make_compressor_sz_generic<T, N>(
make_decomposition_interpolation<T, N>(conf,
LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
HuffmanEncoder<int>(),
Lossless_zstd());
sz->decompress(conf, cmpDataPos, cmpSize, decData);
template <class T, uint N>
size_t SZ_compress_Interp(Config &conf, T *data, uchar *cmpData, size_t cmpCap) {
assert(N == conf.N);
assert(conf.cmprAlgo == ALGO_INTERP);
calAbsErrorBound(conf, data);

auto sz = make_compressor_sz_generic<T, N>(
make_decomposition_interpolation<T, N>(conf, LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
HuffmanEncoder<int>(), Lossless_zstd());
return sz->compress(conf, data, cmpData, cmpCap);
// return cmpData;
}

template <class T, uint N>
void SZ_decompress_Interp(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) {
assert(conf.cmprAlgo == ALGO_INTERP);
auto cmpDataPos = cmpData;
auto sz = make_compressor_sz_generic<T, N>(
make_decomposition_interpolation<T, N>(conf, LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
HuffmanEncoder<int>(), Lossless_zstd());
sz->decompress(conf, cmpDataPos, cmpSize, decData);
}

template <class T, uint N>
double do_not_use_this_interp_compress_block_test(T *data, std::vector<size_t> dims, size_t num, double eb,
int interp_op, int direction_op, int block_size, uchar *buffer,
size_t bufferCap) {
std::vector<T> data1(data, data + num);

Config conf;
conf.absErrorBound = eb;
conf.setDims(dims.begin(), dims.end());
conf.blockSize = block_size;
conf.interpAlgo = interp_op;
conf.interpDirection = direction_op;
auto sz = SZBlockInterpolationCompressor<T, N, LinearQuantizer<T>, HuffmanEncoder<int>, Lossless_zstd>(
LinearQuantizer<T>(eb), HuffmanEncoder<int>(), Lossless_zstd());

size_t outSize = sz.compress(conf, data1.data(), buffer, bufferCap);

auto compression_ratio = num * sizeof(T) * 1.0 / outSize;
return compression_ratio;
}

template <class T, uint N>
size_t SZ_compress_Interp_lorenzo(Config &conf, T *data, uchar *cmpData, size_t cmpCap) {
assert(conf.cmprAlgo == ALGO_INTERP_LORENZO);

// Timer timer(true);

calAbsErrorBound(conf, data);

size_t sampling_num, sampling_block;
std::vector<size_t> sample_dims(N);
std::vector<T> sampling_data = sampling<T, N>(data, conf.dims, sampling_num, sample_dims, sampling_block);
if (sampling_num == conf.num) {
conf.cmprAlgo = ALGO_INTERP;
return SZ_compress_Interp<T, N>(conf, data, cmpData, cmpCap);
}

template<class T, uint N>
double do_not_use_this_interp_compress_block_test(T *data, std::vector<size_t> dims, size_t num,
double eb, int interp_op, int direction_op, int block_size, uchar* buffer, size_t bufferCap) {

std::vector<T> data1(data, data + num);

Config conf;
conf.absErrorBound = eb;
conf.setDims(dims.begin(), dims.end());
conf.blockSize = block_size;
conf.interpAlgo = interp_op;
conf.interpDirection = direction_op;
auto sz = SZBlockInterpolationCompressor<T, N, LinearQuantizer<T>, HuffmanEncoder<int>, Lossless_zstd>(
LinearQuantizer<T>(eb),
HuffmanEncoder<int>(),
Lossless_zstd());

size_t outSize = sz.compress(conf, data1.data(), buffer, bufferCap);

auto compression_ratio = num * sizeof(T) * 1.0 / outSize;
return compression_ratio;

double best_lorenzo_ratio = 0, best_interp_ratio = 0, ratio;
size_t bufferCap = conf.num * sizeof(T);
auto buffer = static_cast<uchar *>(malloc(bufferCap));
Config lorenzo_config = conf;
{
// test lorenzo
lorenzo_config.cmprAlgo = ALGO_LORENZO_REG;
lorenzo_config.setDims(sample_dims.begin(), sample_dims.end());
lorenzo_config.lorenzo = true;
lorenzo_config.lorenzo2 = true;
lorenzo_config.regression = false;
lorenzo_config.regression2 = false;
lorenzo_config.openmp = false;
lorenzo_config.blockSize = 5;
// lorenzo_config.quantbinCnt = 65536 * 2;
std::vector<T> data1(sampling_data);
size_t sampleOutSize = SZ_compress_LorenzoReg<T, N>(lorenzo_config, data1.data(), buffer, bufferCap);
// delete[]cmprData;
// printf("Lorenzo ratio = %.2f\n", ratio);
best_lorenzo_ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize;
}

template<class T, uint N>
size_t SZ_compress_Interp_lorenzo(Config &conf, T *data, uchar *cmpData, size_t cmpCap) {
assert(conf.cmprAlgo == ALGO_INTERP_LORENZO);

// Timer timer(true);

calAbsErrorBound(conf, data);

size_t sampling_num, sampling_block;
std::vector<size_t> sample_dims(N);
std::vector<T> sampling_data = sampling<T, N>(data, conf.dims, sampling_num, sample_dims, sampling_block);
if (sampling_num == conf.num) {
conf.cmprAlgo = ALGO_INTERP;
return SZ_compress_Interp<T, N>(conf, data, cmpData, cmpCap);
}

double best_lorenzo_ratio = 0, best_interp_ratio = 0, ratio;
size_t bufferCap = conf.num * sizeof(T);
auto buffer = (uchar *) malloc(bufferCap);
Config lorenzo_config = conf;
{
//test lorenzo
lorenzo_config.cmprAlgo = ALGO_LORENZO_REG;
lorenzo_config.setDims(sample_dims.begin(), sample_dims.end());
lorenzo_config.lorenzo = true;
lorenzo_config.lorenzo2 = true;
lorenzo_config.regression = false;
lorenzo_config.regression2 = false;
lorenzo_config.openmp = false;
lorenzo_config.blockSize = 5;
// lorenzo_config.quantbinCnt = 65536 * 2;
std::vector<T> data1(sampling_data);
size_t sampleOutSize = SZ_compress_LorenzoReg<T, N>(lorenzo_config, data1.data(), buffer, bufferCap);
// delete[]cmprData;
// printf("Lorenzo ratio = %.2f\n", ratio);
best_lorenzo_ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize;
}

{
//tune interp
for (auto &interp_op : {INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC}) {
ratio = do_not_use_this_interp_compress_block_test<T, N>(sampling_data.data(), sample_dims, sampling_num, conf.absErrorBound,
interp_op, conf.interpDirection, sampling_block, buffer, bufferCap);
if (ratio > best_interp_ratio) {
best_interp_ratio = ratio;
conf.interpAlgo = interp_op;
}
}

int direction_op = factorial(N) - 1;
ratio = do_not_use_this_interp_compress_block_test<T, N>(sampling_data.data(), sample_dims, sampling_num, conf.absErrorBound,
conf.interpAlgo, direction_op, sampling_block, buffer, bufferCap);
if (ratio > best_interp_ratio * 1.02) {

{
// tune interp
for (auto &interp_op : {INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC}) {
ratio = do_not_use_this_interp_compress_block_test<T, N>(
sampling_data.data(), sample_dims, sampling_num, conf.absErrorBound, interp_op, conf.interpDirection,
sampling_block, buffer, bufferCap);
if (ratio > best_interp_ratio) {
best_interp_ratio = ratio;
conf.interpDirection = direction_op;
conf.interpAlgo = interp_op;
}
}

bool useInterp = !(best_lorenzo_ratio > best_interp_ratio && best_lorenzo_ratio < 80 && best_interp_ratio < 80);
size_t cmpSize = 0;
if (useInterp) {
conf.cmprAlgo = ALGO_INTERP;
cmpSize = SZ_compress_Interp<T, N>(conf, data, cmpData, cmpCap);
} else {
//further tune lorenzo
if (N == 3) {
float pred_freq, mean_freq;
T mean_guess;
lorenzo_config.quantbinCnt = optimize_quant_invl_3d<T>(data, conf.dims[0], conf.dims[1], conf.dims[2],
conf.absErrorBound, pred_freq, mean_freq, mean_guess);
lorenzo_config.pred_dim = 2;
size_t sampleOutSize = SZ_compress_LorenzoReg<T, N>(lorenzo_config, sampling_data.data(), buffer, bufferCap);
ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize;
if (ratio > best_lorenzo_ratio * 1.02) {
best_lorenzo_ratio = ratio;
} else {
lorenzo_config.pred_dim = 3;
}

int direction_op = factorial(N) - 1;
ratio = do_not_use_this_interp_compress_block_test<T, N>(sampling_data.data(), sample_dims, sampling_num,
conf.absErrorBound, conf.interpAlgo, direction_op,
sampling_block, buffer, bufferCap);
if (ratio > best_interp_ratio * 1.02) {
best_interp_ratio = ratio;
conf.interpDirection = direction_op;
}
}

bool useInterp = !(best_lorenzo_ratio > best_interp_ratio && best_lorenzo_ratio < 80 && best_interp_ratio < 80);
size_t cmpSize = 0;
if (useInterp) {
conf.cmprAlgo = ALGO_INTERP;
cmpSize = SZ_compress_Interp<T, N>(conf, data, cmpData, cmpCap);
} else {
// further tune lorenzo
if (N == 3) {
float pred_freq, mean_freq;
T mean_guess;
lorenzo_config.quantbinCnt = optimize_quant_invl_3d<T>(
data, conf.dims[0], conf.dims[1], conf.dims[2], conf.absErrorBound, pred_freq, mean_freq, mean_guess);
lorenzo_config.pred_dim = 2;
size_t sampleOutSize =
SZ_compress_LorenzoReg<T, N>(lorenzo_config, sampling_data.data(), buffer, bufferCap);
ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize;
if (ratio > best_lorenzo_ratio * 1.02) {
best_lorenzo_ratio = ratio;
} else {
lorenzo_config.pred_dim = 3;
}

if (conf.relErrorBound < 1.01e-6 && best_lorenzo_ratio > 5 && lorenzo_config.quantbinCnt != 16384) {
auto quant_num = lorenzo_config.quantbinCnt;
lorenzo_config.quantbinCnt = 16384;
size_t sampleOutSize = SZ_compress_LorenzoReg<T, N>(lorenzo_config, sampling_data.data(), buffer, bufferCap);
// delete[]cmprData;
ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize;
if (ratio > best_lorenzo_ratio * 1.02) {
best_lorenzo_ratio = ratio;
} else {
lorenzo_config.quantbinCnt = quant_num;
}
}

if (conf.relErrorBound < 1.01e-6 && best_lorenzo_ratio > 5 && lorenzo_config.quantbinCnt != 16384) {
auto quant_num = lorenzo_config.quantbinCnt;
lorenzo_config.quantbinCnt = 16384;
size_t sampleOutSize =
SZ_compress_LorenzoReg<T, N>(lorenzo_config, sampling_data.data(), buffer, bufferCap);
// delete[]cmprData;
ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize;
if (ratio > best_lorenzo_ratio * 1.02) {
best_lorenzo_ratio = ratio;
} else {
lorenzo_config.quantbinCnt = quant_num;
}
lorenzo_config.setDims(conf.dims.begin(), conf.dims.end());
conf = lorenzo_config;
// double tuning_time = timer.stop();
cmpSize = SZ_compress_LorenzoReg<T, N>(conf, data, cmpData, cmpCap);
}

free(buffer);
return cmpSize;
lorenzo_config.setDims(conf.dims.begin(), conf.dims.end());
conf = lorenzo_config;
// double tuning_time = timer.stop();
cmpSize = SZ_compress_LorenzoReg<T, N>(conf, data, cmpData, cmpCap);
}

free(buffer);
return cmpSize;
}
} // namespace SZ3
#endif
Loading

0 comments on commit 876a3ca

Please sign in to comment.