Skip to content

Commit

Permalink
change namespace from SZ to SZ3, to avoid conflict with SZ2
Browse files Browse the repository at this point in the history
  • Loading branch information
@ayzk
ayzk committed Oct 31, 2023
1 parent e1af6be commit aadaaeb
Show file tree
Hide file tree
Showing 10 changed files with 133 additions and 133 deletions.
2 changes: 1 addition & 1 deletion include/SZ3/utils/FileUtil.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ namespace SZ3 {
fin.seekg(0, std::ios::end);
const size_t num_elements = fin.tellg() / sizeof(Type);
fin.seekg(0, std::ios::beg);
// auto data = SZ::compat::make_unique<Type[]>(num_elements);
// auto data = SZ3::compat::make_unique<Type[]>(num_elements);
auto data = std::make_unique<Type[]>(num_elements);
fin.read(reinterpret_cast<char *>(&data[0]), num_elements * sizeof(Type));
fin.close();
Expand Down
28 changes: 14 additions & 14 deletions tools/H5Z-SZ3/src/H5Z_SZ3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigne
{

/* decompress data */
SZ::Config conf;
SZ3::Config conf;

switch(dataType) {
case SZ_FLOAT: //FLOAT
Expand Down Expand Up @@ -493,20 +493,20 @@ static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigne
exit(0);
}

SZ::Config conf;
SZ3::Config conf;
//printf("\nDIMS_CMP:\n");
//printf("r1 %u r2 %u r3 %u r4 %u r5 %u\n", r1,r2,r3,r4,r5);
if (r2 == 0) {
conf = SZ::Config(r1);
conf = SZ3::Config(r1);
} else if (r3 == 0) {
conf = SZ::Config(r2, r1);
conf = SZ3::Config(r2, r1);
} else if (r4 == 0) {
conf = SZ::Config(r3, r2, r1);
conf = SZ3::Config(r3, r2, r1);
} else if (r5 == 0) {
conf = SZ::Config(r4, r3, r2, r1);
conf = SZ3::Config(r4, r3, r2, r1);
}
else {
conf = SZ::Config(r5, r4, r3, r2, r1);
conf = SZ3::Config(r5, r4, r3, r2, r1);
}

if(error_mode < 0 || error_mode > 5) {
Expand Down Expand Up @@ -639,12 +639,12 @@ void SZ_cdArrayToMetaData(size_t cd_nelmts, const unsigned int cd_values[], int*
switch(*dimSize)
{
case 1:
SZ::int32ToBytes_bigEndian(bytes, cd_values[2]);
SZ::int32ToBytes_bigEndian(&bytes[4], cd_values[3]);
SZ3::int32ToBytes_bigEndian(bytes, cd_values[2]);
SZ3::int32ToBytes_bigEndian(&bytes[4], cd_values[3]);
if(sizeof(size_t)==4)
*r1 = (unsigned int) SZ::bytesToInt64_bigEndian(bytes);
*r1 = (unsigned int) SZ3::bytesToInt64_bigEndian(bytes);
else
*r1 = (uint64_t) SZ::bytesToInt64_bigEndian(bytes);
*r1 = (uint64_t) SZ3::bytesToInt64_bigEndian(bytes);
*r2 = *r3 = *r4 = *r5 = 0;
break;
case 2:
Expand Down Expand Up @@ -710,9 +710,9 @@ void SZ_copymetaDataToCdArray(size_t* cd_nelmts, unsigned int *cd_values, int da
{
case 1:
size = (uint64_t) r1;
SZ::int64ToBytes_bigEndian(bytes, size);
cd_values[2] = SZ::bytesToInt32_bigEndian(bytes);
cd_values[3] = SZ::bytesToInt32_bigEndian(&bytes[4]);
SZ3::int64ToBytes_bigEndian(bytes, size);
cd_values[2] = SZ3::bytesToInt32_bigEndian(bytes);
cd_values[3] = SZ3::bytesToInt32_bigEndian(&bytes[4]);
*cd_nelmts = 4;
break;
case 2:
Expand Down
2 changes: 1 addition & 1 deletion tools/mdz/include/KmeansUtil.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ namespace SZ3 {
size_t sample_num) {
T max = *std::max_element(data, data + num);
T min = *std::min_element(data, data + num);
SZ::Timer timer;
SZ3::Timer timer;
timer.start();
std::vector<T> sample;
if (num == sample_num) {
Expand Down
8 changes: 4 additions & 4 deletions tools/mdz/include/TimeBasedFrontend.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ namespace SZ3 {
global_dims[i] = global_dimensions[i + 1];
};

auto inter_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>(
auto inter_block_range = std::make_shared<SZ3::multi_dimensional_range<T, N - 1>>(
data, std::begin(global_dims), std::end(global_dims), stride, 0);
auto intra_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>(
auto intra_block_range = std::make_shared<SZ3::multi_dimensional_range<T, N - 1>>(
data, std::begin(global_dims), std::end(global_dims), 1, 0);

// std::array<size_t, N - 1> intra_block_dims;
Expand Down Expand Up @@ -114,10 +114,10 @@ namespace SZ3 {
for (int i = 0; i < N - 1; i++) {
global_dims[i] = global_dimensions[i + 1];
};
auto inter_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>(
auto inter_block_range = std::make_shared<SZ3::multi_dimensional_range<T, N - 1>>(
dec_data, std::begin(global_dims), std::end(global_dims), stride, 0);

auto intra_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>(
auto intra_block_range = std::make_shared<SZ3::multi_dimensional_range<T, N - 1>>(
dec_data, std::begin(global_dims), std::end(global_dims), 1, 0);

predictor.predecompress_data(inter_block_range->begin());
Expand Down
126 changes: 63 additions & 63 deletions tools/mdz/include/mdz.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,110 +29,110 @@ const char *compressor_names[] = {"VQ", "VQT", "MT", "LR", "TS"};


template<typename T, uint N, class Predictor>
SZ::concepts::CompressorInterface<T> *
make_sz_timebased2(const SZ::Config &conf, Predictor predictor, T *data_ts0) {
return new SZ::SZGeneralCompressor<T, N, SZ::TimeBasedFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>,
SZ::HuffmanEncoder<int>, SZ::Lossless_zstd>(
SZ::TimeBasedFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>(conf, predictor,
SZ::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2),
SZ3::concepts::CompressorInterface<T> *
make_sz_timebased2(const SZ3::Config &conf, Predictor predictor, T *data_ts0) {
return new SZ3::SZGeneralCompressor<T, N, SZ3::TimeBasedFrontend<T, N, Predictor, SZ3::LinearQuantizer<T>>,
SZ3::HuffmanEncoder<int>, SZ3::Lossless_zstd>(
SZ3::TimeBasedFrontend<T, N, Predictor, SZ3::LinearQuantizer<T>>(conf, predictor,
SZ3::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2),
data_ts0),
SZ::HuffmanEncoder<int>(),
SZ::Lossless_zstd());
SZ3::HuffmanEncoder<int>(),
SZ3::Lossless_zstd());
}

template<typename T, uint N>
SZ::concepts::CompressorInterface<T> *
make_sz_timebased(const SZ::Config &conf, T *data_ts0) {
std::vector<std::shared_ptr<SZ::concepts::PredictorInterface<T, N - 1>>> predictors;
SZ3::concepts::CompressorInterface<T> *
make_sz_timebased(const SZ3::Config &conf, T *data_ts0) {
std::vector<std::shared_ptr<SZ3::concepts::PredictorInterface<T, N - 1>>> predictors;

int use_single_predictor =
(conf.lorenzo + conf.lorenzo2 + conf.regression) == 1;
if (conf.lorenzo) {
if (use_single_predictor) {
return make_sz_timebased2<T, N>(conf, SZ::LorenzoPredictor<T, N - 1, 1>(conf.absErrorBound), data_ts0);
return make_sz_timebased2<T, N>(conf, SZ3::LorenzoPredictor<T, N - 1, 1>(conf.absErrorBound), data_ts0);
} else {
predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N - 1, 1>>(conf.absErrorBound));
predictors.push_back(std::make_shared<SZ3::LorenzoPredictor<T, N - 1, 1>>(conf.absErrorBound));
}
}
if (conf.lorenzo2) {
if (use_single_predictor) {
return make_sz_timebased2<T, N>(conf, SZ::LorenzoPredictor<T, N - 1, 2>(conf.absErrorBound), data_ts0);
return make_sz_timebased2<T, N>(conf, SZ3::LorenzoPredictor<T, N - 1, 2>(conf.absErrorBound), data_ts0);
} else {
predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N - 1, 2>>(conf.absErrorBound));
predictors.push_back(std::make_shared<SZ3::LorenzoPredictor<T, N - 1, 2>>(conf.absErrorBound));
}
}
if (conf.regression) {
if (use_single_predictor) {
return make_sz_timebased2<T, N>(conf, SZ::RegressionPredictor<T, N - 1>(conf.blockSize, conf.absErrorBound), data_ts0);
return make_sz_timebased2<T, N>(conf, SZ3::RegressionPredictor<T, N - 1>(conf.blockSize, conf.absErrorBound), data_ts0);
} else {
predictors.push_back(std::make_shared<SZ::RegressionPredictor<T, N - 1>>(conf.blockSize, conf.absErrorBound));
predictors.push_back(std::make_shared<SZ3::RegressionPredictor<T, N - 1>>(conf.blockSize, conf.absErrorBound));
}
}

return make_sz_timebased2<T, N>(conf, SZ::ComposedPredictor<T, N - 1>(predictors), data_ts0);
return make_sz_timebased2<T, N>(conf, SZ3::ComposedPredictor<T, N - 1>(predictors), data_ts0);
}

template<typename T, uint N, class Predictor>
SZ::concepts::CompressorInterface<T> *
make_sz2(const SZ::Config &conf, Predictor predictor) {

return new SZ::SZGeneralCompressor<T, N, SZ::SZGeneralFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>,
SZ::HuffmanEncoder<int>, SZ::Lossless_zstd>(
SZ::SZGeneralFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>(conf, predictor,
SZ::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
SZ::HuffmanEncoder<int>(),
SZ::Lossless_zstd());
SZ3::concepts::CompressorInterface<T> *
make_sz2(const SZ3::Config &conf, Predictor predictor) {

return new SZ3::SZGeneralCompressor<T, N, SZ3::SZGeneralFrontend<T, N, Predictor, SZ3::LinearQuantizer<T>>,
SZ3::HuffmanEncoder<int>, SZ3::Lossless_zstd>(
SZ3::SZGeneralFrontend<T, N, Predictor, SZ3::LinearQuantizer<T>>(conf, predictor,
SZ3::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
SZ3::HuffmanEncoder<int>(),
SZ3::Lossless_zstd());
}

template<typename T, uint N>
SZ::concepts::CompressorInterface<T> *
make_sz(const SZ::Config &conf) {
std::vector<std::shared_ptr<SZ::concepts::PredictorInterface<T, N>>> predictors;
SZ3::concepts::CompressorInterface<T> *
make_sz(const SZ3::Config &conf) {
std::vector<std::shared_ptr<SZ3::concepts::PredictorInterface<T, N>>> predictors;

int use_single_predictor =
(conf.lorenzo + conf.lorenzo2 + conf.regression) == 1;
if (conf.lorenzo) {
if (use_single_predictor) {
return make_sz2<T, N>(conf, SZ::LorenzoPredictor<T, N, 1>(conf.absErrorBound));
return make_sz2<T, N>(conf, SZ3::LorenzoPredictor<T, N, 1>(conf.absErrorBound));
} else {
predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N, 1>>(conf.absErrorBound));
predictors.push_back(std::make_shared<SZ3::LorenzoPredictor<T, N, 1>>(conf.absErrorBound));
}
}
if (conf.lorenzo2) {
if (use_single_predictor) {
return make_sz2<T, N>(conf, SZ::LorenzoPredictor<T, N, 2>(conf.absErrorBound));
return make_sz2<T, N>(conf, SZ3::LorenzoPredictor<T, N, 2>(conf.absErrorBound));
} else {
predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N, 2>>(conf.absErrorBound));
predictors.push_back(std::make_shared<SZ3::LorenzoPredictor<T, N, 2>>(conf.absErrorBound));
}
}
if (conf.regression) {
if (use_single_predictor) {
return make_sz2<T, N>(conf, SZ::RegressionPredictor<T, N>(conf.blockSize, conf.absErrorBound));
return make_sz2<T, N>(conf, SZ3::RegressionPredictor<T, N>(conf.blockSize, conf.absErrorBound));
} else {
predictors.push_back(std::make_shared<SZ::RegressionPredictor<T, N >>(conf.blockSize, conf.absErrorBound));
predictors.push_back(std::make_shared<SZ3::RegressionPredictor<T, N >>(conf.blockSize, conf.absErrorBound));
}
}

return make_sz2<T, N>(conf, SZ::ComposedPredictor<T, N>(predictors));
return make_sz2<T, N>(conf, SZ3::ComposedPredictor<T, N>(predictors));
}


template<typename T, uint N>
float *
VQ(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom,
VQ(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom,
int method, float level_start, float level_offset, int level_num) {
if (level_num == 0) {
printf("VQ/VQT not availble on current dataset, please use ADP or MT\n");
exit(0);
}

auto sz = SZ::SZ_Exaalt_Compressor<float, N, SZ::LinearQuantizer<float>, SZ::HuffmanEncoder<int>,
SZ::Lossless_zstd>(conf, SZ::LinearQuantizer<float>(conf.absErrorBound, conf.quantbinCnt / 2),
SZ::HuffmanEncoder<int>(), SZ::Lossless_zstd(), method);
auto sz = SZ3::SZ_Exaalt_Compressor<float, N, SZ3::LinearQuantizer<float>, SZ3::HuffmanEncoder<int>,
SZ3::Lossless_zstd>(conf, SZ3::LinearQuantizer<float>(conf.absErrorBound, conf.quantbinCnt / 2),
SZ3::HuffmanEncoder<int>(), SZ3::Lossless_zstd(), method);
sz.set_level(level_start, level_offset, level_num);

SZ::Timer timer(true);
SZ::uchar *compressed;
SZ3::Timer timer(true);
SZ3::uchar *compressed;
compressed = sz.compress(data, compressed_size);
total_compress_time += timer.stop("Compression");
if (!decom) {
Expand All @@ -154,12 +154,12 @@ VQ(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom,

template<typename T, uint N>
float *
MT(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *ts0) {
MT(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *ts0) {
// printf("eb=%.8f\n", conf.eb);
auto sz = make_sz_timebased<T, N>(conf, ts0);

SZ::uchar *compressed;
SZ::Timer timer(true);
SZ3::uchar *compressed;
SZ3::Timer timer(true);
compressed = sz->compress(conf, data, compressed_size);
total_compress_time += timer.stop("Compression");
if (!decom) {
Expand All @@ -183,11 +183,11 @@ MT(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *

template<typename T, uint N>
float *
SZ2(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) {
SZ2(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) {
auto sz = make_sz<T, N>(conf);
SZ::uchar *compressed;
SZ3::uchar *compressed;

SZ::Timer timer(true);
SZ3::Timer timer(true);
compressed = sz->compress(conf, data, compressed_size);
total_compress_time += timer.stop("Compression");
if (!decom) {
Expand All @@ -210,7 +210,7 @@ SZ2(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) {


template<typename T, uint N>
void select(SZ::Config conf, int &method, size_t ts, T *data_all,
void select(SZ3::Config conf, int &method, size_t ts, T *data_all,
float level_start, float level_offset, int level_num, T *data_ts0, size_t timestep_batch) {
// && (ts_last_select == -1 || t - ts_last_select >= conf.timestep_batch * 10)) {
// std::cout << "****************** BEGIN Selection ****************" << std::endl;
Expand Down Expand Up @@ -273,17 +273,17 @@ std::unique_ptr<Type[]> readfile(const char *file, size_t start, size_t num) {


template<typename T, uint N>
SZ::uchar *LAMMPS_compress(SZ::Config conf, T *data, int method, size_t &compressed_size,
SZ3::uchar *LAMMPS_compress(SZ3::Config conf, T *data, int method, size_t &compressed_size,
float level_start, float level_offset, int level_num, T *ts0) {
if ((method == 0 || method == 1) && level_num == 0) {
printf("VQ/VQT not available on current dataset, please use ADP or MT\n");
exit(0);
}
SZ::uchar *compressed_data;
SZ3::uchar *compressed_data;
if (method == 0 || method == 1) {
auto sz = SZ::SZ_Exaalt_Compressor<T, N, SZ::LinearQuantizer<float>, SZ::HuffmanEncoder<int>, SZ::Lossless_zstd>(
conf, SZ::LinearQuantizer<float>(conf.absErrorBound, conf.quantbinCnt / 2),
SZ::HuffmanEncoder<int>(), SZ::Lossless_zstd(), method);
auto sz = SZ3::SZ_Exaalt_Compressor<T, N, SZ3::LinearQuantizer<float>, SZ3::HuffmanEncoder<int>, SZ3::Lossless_zstd>(
conf, SZ3::LinearQuantizer<float>(conf.absErrorBound, conf.quantbinCnt / 2),
SZ3::HuffmanEncoder<int>(), SZ3::Lossless_zstd(), method);
sz.set_level(level_start, level_offset, level_num);
compressed_data = sz.compress(data, compressed_size);
} else if (method == 2 || method == 4) {
Expand All @@ -300,7 +300,7 @@ SZ::uchar *LAMMPS_compress(SZ::Config conf, T *data, int method, size_t &compres
}

template<typename T, uint N>
int LAMMPS_select_compressor(SZ::Config conf, T *data, bool firsttime,
int LAMMPS_select_compressor(SZ3::Config conf, T *data, bool firsttime,
float level_start, float level_offset, int level_num, T *data_ts0) {
// std::cout << "****************** BEGIN Selection ****************" << std::endl;

Expand All @@ -317,7 +317,7 @@ int LAMMPS_select_compressor(SZ::Config conf, T *data, bool firsttime,
}

std::vector<T> data1;
SZ::uchar *cmpr;
SZ3::uchar *cmpr;
if (level_num > 0) {

data1 = std::vector<T>(data, data + conf.num);
Expand Down Expand Up @@ -350,7 +350,7 @@ int LAMMPS_select_compressor(SZ::Config conf, T *data, bool firsttime,

template<typename T, uint N>
inline typename std::enable_if<N == 1 || N == 2, size_t>::type
MDZ_Compress(SZ::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) {
MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) {
// if (N != 2) {
// throw std::invalid_argument("dimension should be 2");
// }
Expand Down Expand Up @@ -382,7 +382,7 @@ MDZ_Compress(SZ::Config conf, T *input_data, T *dec_data, size_t batch_size, int
size_t sample_num = 0.1 * conf.dims[1];
sample_num = std::min(sample_num, (size_t) 20000);
sample_num = std::max(sample_num, std::min((size_t) 5000, conf.dims[1]));
SZ::get_cluster(input_data, conf.dims[1], level_start, level_offset, level_num,
SZ3::get_cluster(input_data, conf.dims[1], level_start, level_offset, level_num,
sample_num);
if (level_num > conf.dims[1] * 0.25) {
level_num = 0;
Expand All @@ -405,9 +405,9 @@ MDZ_Compress(SZ::Config conf, T *input_data, T *dec_data, size_t batch_size, int

T max = *std::max_element(data, data + conf.num);
T min = *std::min_element(data, data + conf.num);
if (conf.errorBoundMode == SZ::EB_ABS) {
if (conf.errorBoundMode == SZ3::EB_ABS) {
conf.relErrorBound = conf.absErrorBound / (max - min);
} else if (conf.errorBoundMode == SZ::EB_REL) {
} else if (conf.errorBoundMode == SZ3::EB_REL) {
conf.absErrorBound = conf.relErrorBound * (max - min);
}

Expand Down Expand Up @@ -446,7 +446,7 @@ MDZ_Compress(SZ::Config conf, T *input_data, T *dec_data, size_t batch_size, int
}

template<typename T, uint N>
inline typename std::enable_if<N == 3, size_t>::type MDZ_Compress(SZ::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) {
inline typename std::enable_if<N == 3, size_t>::type MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) {
size_t total_compressed_size = 0;
auto dims = conf.dims;
std::vector<T> input(conf.num), output(conf.num);
Expand All @@ -458,7 +458,7 @@ inline typename std::enable_if<N == 3, size_t>::type MDZ_Compress(SZ::Config con
}
}
for (int i = 0; i < conf.dims[2]; i++) {
SZ::Config conf_2D(conf);
SZ3::Config conf_2D(conf);
conf_2D.dims = {conf.dims[0], conf.dims[1]};
conf_2D.num = conf.dims[0] * conf.dims[1];
total_compressed_size += MDZ_Compress<T, 2>(conf_2D, input.data() + i * conf.dims[0] * conf.dims[1],
Expand Down
Loading

0 comments on commit aadaaeb

Please sign in to comment.