diff --git a/CMakeLists.txt b/CMakeLists.txt index c1cddbc1..38e827fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,11 @@ cmake_minimum_required(VERSION 3.18) -project(SZ3 VERSION 3.1.8) +project(SZ3 VERSION 3.2.0) + +#data version defines the version of the compressed data format +#it is not always equal to the program version (e.g., SZ3 v3.1.0 and SZ3 v.3.1.1 may use the same data version of v.3.1.0) +#only update data version if the new version of the program changes compressed data format +set(SZ3_DATA_VERSION 3.2.0) + include(GNUInstallDirs) include(CTest) @@ -19,8 +25,9 @@ endif () find_package(PkgConfig) - -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp) +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in + ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp) add_library(${PROJECT_NAME} INTERFACE) target_include_directories( @@ -31,7 +38,7 @@ target_include_directories( ) target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_17 - ) +) find_package(OpenMP) if (OpenMP_FOUND) @@ -84,18 +91,18 @@ install(TARGETS ${PROJECT_NAME} EXPORT SZ3Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - ) +) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SZ3/" - ) +) install(EXPORT SZ3Targets NAMESPACE SZ3:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3) include(CMakePackageConfigHelpers) configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/SZ3Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake" INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3 - ) +) write_basic_package_version_file( "${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake" VERSION ${PROJECT_VERSION} @@ -105,6 +112,6 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake ${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3 - ) +) #export sz3 target for external use export(TARGETS SZ3 FILE SZ3.cmake) diff --git a/README.md b/README.md index c6b17752..b1c42e98 100644 --- a/README.md +++ b/README.md @@ -62,17 +62,20 @@ Scripts without parameters below should work fine by replacing SZ2 with SZ3. * Visit [this Github repository](https://github.com/ofmla/sz3_simple_example) for details #### H5Z-SZ3 +* Located in 'tools/H5Z-SZ3' +* Please add "-DBUILD_H5Z_FILTER=ON" to enable this function for CMake. +* sz3ToHDF5 and HDF5ToSz3 are provided for testing. -* Use examples/print_h5repack_args.c to construct the cd_values parameters based on the specified error configuration. - -* Compression example: -`h5repack -f UD=32024,0,5,0,981668463,0,0,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5` - -* Decompression example: -`h5repack -f NONE -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.out.h5` - -* Alternatively, the error bound information can also be given through sz3.config (when there are no cd_values for h5repack). Example (You need to put sz3.config in the current local directory so that it will read sz3.config to get error bounds): -`h5repack -f UD=32024,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5` +[//]: # (* Use examples/print_h5repack_args.c to construct the cd_values parameters based on the specified error configuration.) +[//]: # () +[//]: # (* Compression example: ) +[//]: # (`h5repack -f UD=32024,0,5,0,981668463,0,0,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5`) +[//]: # () +[//]: # (* Decompression example:) +[//]: # (`h5repack -f NONE -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.out.h5`) +[//]: # () +[//]: # (* Alternatively, the error bound information can also be given through sz3.config (when there are no cd_values for h5repack). Example (You need to put sz3.config in the current local directory so that it will read sz3.config to get error bounds):) +[//]: # (`h5repack -f UD=32024,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5`) @@ -91,6 +94,8 @@ Version New features * SZ 3.1.6 Support C API and Python API. * SZ 3.1.7 Initial MDZ(https://github.com/szcompressor/SZ3/tree/master/tools/mdz) support. * SZ 3.1.8 namespace changed from SZ to SZ3. H5Z-SZ3 supports configuration file now. +* SZ 3.2.0 API reconstructed for FZ. H5Z-SZ3 rewrite. Compression version checking. + ## Citations **Kindly note**: If you mention SZ in your paper, the most appropriate citation is including these three references (**TBD22, ICDE21, Bigdata18**), because they cover the design and implementation of the latest version of SZ. diff --git a/include/SZ3/api/impl/SZAlgo.hpp b/include/SZ3/api/impl/SZAlgo.hpp new file mode 100644 index 00000000..59e34d9d --- /dev/null +++ b/include/SZ3/api/impl/SZAlgo.hpp @@ -0,0 +1,42 @@ +#ifndef SZ3_SZALGO_HPP +#define SZ3_SZALGO_HPP + +#include "SZ3/compressor/SZGenericCompressor.hpp" +#include "SZ3/decomposition/NoPredictionDecomposition.hpp" +#include "SZ3/quantizer/IntegerQuantizer.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/utils/Config.hpp" + +namespace SZ3 { + template + size_t SZ_compress_nopred(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); + assert(conf.cmprAlgo == ALGO_INTERP); + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_generic( + make_decomposition_noprediction(conf, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + return sz->compress(conf, data, cmpData, cmpCap); +// return cmpData; + } + + + template + void SZ_decompress_nopred(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + assert(conf.cmprAlgo == ALGO_INTERP); + auto cmpDataPos = cmpData; + auto sz = make_compressor_sz_generic( + make_decomposition_noprediction(conf, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); + } + + +} +#endif diff --git a/include/SZ3/api/impl/SZInterp.hpp b/include/SZ3/api/impl/SZAlgoInterp.hpp similarity index 62% rename from include/SZ3/api/impl/SZInterp.hpp rename to include/SZ3/api/impl/SZAlgoInterp.hpp index 97a7ad80..2aa7f68c 100644 --- a/include/SZ3/api/impl/SZInterp.hpp +++ b/include/SZ3/api/impl/SZAlgoInterp.hpp @@ -1,8 +1,8 @@ -#ifndef SZ3_SZINTERP_HPP -#define SZ3_SZINTERP_HPP +#ifndef SZ3_SZALGOINTERP_HPP +#define SZ3_SZALGOINTERP_HPP -#include "SZ3/compressor/SZInterpolationCompressor.hpp" -#include "SZ3/compressor/deprecated/SZBlockInterpolationCompressor.hpp" +#include "SZ3/decomposition/InterpolationDecomposition.hpp" +#include "SZ3/compressor/specialized/SZBlockInterpolationCompressor.hpp" #include "SZ3/quantizer/IntegerQuantizer.hpp" #include "SZ3/lossless/Lossless_zstd.hpp" #include "SZ3/utils/Iterator.hpp" @@ -10,47 +10,44 @@ #include "SZ3/utils/Extraction.hpp" #include "SZ3/utils/QuantOptimizatioin.hpp" #include "SZ3/utils/Config.hpp" -#include "SZ3/api/impl/SZLorenzoReg.hpp" +#include "SZ3/api/impl/SZAlgoLorenzoReg.hpp" #include #include namespace SZ3 { template - char *SZ_compress_Interp(Config &conf, T *data, size_t &outSize) { - - + size_t SZ_compress_Interp(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { assert(N == conf.N); assert(conf.cmprAlgo == ALGO_INTERP); calAbsErrorBound(conf, data); - - auto sz = SZInterpolationCompressor, HuffmanEncoder, Lossless_zstd>( - LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), - HuffmanEncoder(), - Lossless_zstd()); - char *cmpData = (char *) sz.compress(conf, data, outSize); - return cmpData; + + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + return sz->compress(conf, data, cmpData, cmpCap); +// return cmpData; } - - + template - void SZ_decompress_Interp(const Config &conf, char *cmpData, size_t cmpSize, T *decData) { + void SZ_decompress_Interp(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { assert(conf.cmprAlgo == ALGO_INTERP); - uchar const *cmpDataPos = (uchar *) cmpData; - auto sz = SZInterpolationCompressor, HuffmanEncoder, Lossless_zstd>( - LinearQuantizer(), - HuffmanEncoder(), - Lossless_zstd()); - sz.decompress(cmpDataPos, cmpSize, decData); + auto cmpDataPos = cmpData; + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); } - - + template double do_not_use_this_interp_compress_block_test(T *data, std::vector dims, size_t num, - double eb, int interp_op, int direction_op, int block_size) { - + double eb, int interp_op, int direction_op, int block_size, uchar* buffer, size_t bufferCap) { + std::vector data1(data, data + num); - size_t outSize = 0; - + Config conf; conf.absErrorBound = eb; conf.setDims(dims.begin(), dims.end()); @@ -58,34 +55,35 @@ namespace SZ3 { conf.interpAlgo = interp_op; conf.interpDirection = direction_op; auto sz = SZBlockInterpolationCompressor, HuffmanEncoder, Lossless_zstd>( - LinearQuantizer(eb), - HuffmanEncoder(), - Lossless_zstd()); - char *cmpData = (char *) sz.compress(conf, data1.data(), outSize); - delete[]cmpData; + LinearQuantizer(eb), + HuffmanEncoder(), + Lossless_zstd()); + + size_t outSize = sz.compress(conf, data1.data(), buffer, bufferCap); + auto compression_ratio = num * sizeof(T) * 1.0 / outSize; return compression_ratio; } - + template - char *SZ_compress_Interp_lorenzo(Config &conf, T *data, size_t &outSize) { + size_t SZ_compress_Interp_lorenzo(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { assert(conf.cmprAlgo == ALGO_INTERP_LORENZO); - - Timer timer(true); - + +// Timer timer(true); + calAbsErrorBound(conf, data); - + size_t sampling_num, sampling_block; std::vector sample_dims(N); std::vector sampling_data = sampling(data, conf.dims, sampling_num, sample_dims, sampling_block); if (sampling_num == conf.num) { conf.cmprAlgo = ALGO_INTERP; - return SZ_compress_Interp(conf, data, outSize); + return SZ_compress_Interp(conf, data, cmpData, cmpCap); } - + double best_lorenzo_ratio = 0, best_interp_ratio = 0, ratio; - size_t sampleOutSize; - char *cmprData; + size_t bufferCap = conf.num * sizeof(T); + auto buffer = (uchar *) malloc(bufferCap); Config lorenzo_config = conf; { //test lorenzo @@ -99,48 +97,46 @@ namespace SZ3 { lorenzo_config.blockSize = 5; // lorenzo_config.quantbinCnt = 65536 * 2; std::vector data1(sampling_data); - cmprData = SZ_compress_LorenzoReg(lorenzo_config, data1.data(), sampleOutSize); - delete[]cmprData; + size_t sampleOutSize = SZ_compress_LorenzoReg(lorenzo_config, data1.data(), buffer, bufferCap); +// delete[]cmprData; // printf("Lorenzo ratio = %.2f\n", ratio); best_lorenzo_ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize; } - + { //tune interp - for (auto &interp_op: {INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC}) { + for (auto &interp_op : {INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC}) { ratio = do_not_use_this_interp_compress_block_test(sampling_data.data(), sample_dims, sampling_num, conf.absErrorBound, - interp_op, conf.interpDirection, sampling_block); + interp_op, conf.interpDirection, sampling_block, buffer, bufferCap); if (ratio > best_interp_ratio) { best_interp_ratio = ratio; conf.interpAlgo = interp_op; } } - + int direction_op = factorial(N) - 1; ratio = do_not_use_this_interp_compress_block_test(sampling_data.data(), sample_dims, sampling_num, conf.absErrorBound, - conf.interpAlgo, direction_op, sampling_block); + conf.interpAlgo, direction_op, sampling_block, buffer, bufferCap); if (ratio > best_interp_ratio * 1.02) { best_interp_ratio = ratio; conf.interpDirection = direction_op; } } - + bool useInterp = !(best_lorenzo_ratio > best_interp_ratio && best_lorenzo_ratio < 80 && best_interp_ratio < 80); - + size_t cmpSize = 0; if (useInterp) { conf.cmprAlgo = ALGO_INTERP; - double tuning_time = timer.stop(); - return SZ_compress_Interp(conf, data, outSize); + cmpSize = SZ_compress_Interp(conf, data, cmpData, cmpCap); } else { //further tune lorenzo if (N == 3) { float pred_freq, mean_freq; T mean_guess; lorenzo_config.quantbinCnt = optimize_quant_invl_3d(data, conf.dims[0], conf.dims[1], conf.dims[2], - conf.absErrorBound, pred_freq, mean_freq, mean_guess); + conf.absErrorBound, pred_freq, mean_freq, mean_guess); lorenzo_config.pred_dim = 2; - cmprData = SZ_compress_LorenzoReg(lorenzo_config, sampling_data.data(), sampleOutSize); - delete[]cmprData; + size_t sampleOutSize = SZ_compress_LorenzoReg(lorenzo_config, sampling_data.data(), buffer, bufferCap); ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize; if (ratio > best_lorenzo_ratio * 1.02) { best_lorenzo_ratio = ratio; @@ -148,12 +144,12 @@ namespace SZ3 { lorenzo_config.pred_dim = 3; } } - + if (conf.relErrorBound < 1.01e-6 && best_lorenzo_ratio > 5 && lorenzo_config.quantbinCnt != 16384) { auto quant_num = lorenzo_config.quantbinCnt; lorenzo_config.quantbinCnt = 16384; - cmprData = SZ_compress_LorenzoReg(lorenzo_config, sampling_data.data(), sampleOutSize); - delete[]cmprData; + size_t sampleOutSize = SZ_compress_LorenzoReg(lorenzo_config, sampling_data.data(), buffer, bufferCap); +// delete[]cmprData; ratio = sampling_num * 1.0 * sizeof(T) / sampleOutSize; if (ratio > best_lorenzo_ratio * 1.02) { best_lorenzo_ratio = ratio; @@ -163,11 +159,12 @@ namespace SZ3 { } lorenzo_config.setDims(conf.dims.begin(), conf.dims.end()); conf = lorenzo_config; - double tuning_time = timer.stop(); - return SZ_compress_LorenzoReg(conf, data, outSize); +// double tuning_time = timer.stop(); + cmpSize = SZ_compress_LorenzoReg(conf, data, cmpData, cmpCap); } - - + + free(buffer); + return cmpSize; } } #endif diff --git a/include/SZ3/api/impl/SZLorenzoReg.hpp b/include/SZ3/api/impl/SZAlgoLorenzoReg.hpp similarity index 54% rename from include/SZ3/api/impl/SZLorenzoReg.hpp rename to include/SZ3/api/impl/SZAlgoLorenzoReg.hpp index fef0308c..c9ea76d4 100644 --- a/include/SZ3/api/impl/SZLorenzoReg.hpp +++ b/include/SZ3/api/impl/SZAlgoLorenzoReg.hpp @@ -1,9 +1,9 @@ #ifndef SZ3_SZ_LORENZO_REG_HPP #define SZ3_SZ_LORENZO_REG_HPP -#include "SZ3/compressor/SZGeneralCompressor.hpp" -#include "SZ3/frontend/SZFastFrontend.hpp" -#include "SZ3/frontend/SZGeneralFrontend.hpp" +#include "SZ3/compressor/SZIterateCompressor.hpp" +#include "SZ3/compressor/SZGenericCompressor.hpp" +#include "SZ3/decomposition/LorenzoRegressionDecomposition.hpp" #include "SZ3/quantizer/IntegerQuantizer.hpp" #include "SZ3/predictor/ComposedPredictor.hpp" #include "SZ3/predictor/LorenzoPredictor.hpp" @@ -22,7 +22,7 @@ namespace SZ3 { template std::shared_ptr> - make_lorenzo_regression_compressor(const Config &conf, Quantizer quantizer, Encoder encoder, Lossless lossless) { + make_compressor_typetwo_lorenzo_regression(const Config &conf, Quantizer quantizer, Encoder encoder, Lossless lossless) { std::vector>> predictors; int methodCnt = (conf.lorenzo + conf.lorenzo2 + conf.regression + conf.regression2); @@ -33,27 +33,26 @@ namespace SZ3 { } if (conf.lorenzo) { if (use_single_predictor) { - return make_sz_general_compressor( - make_sz_general_frontend(conf, LorenzoPredictor(conf.absErrorBound), quantizer), - encoder, lossless); + return make_compressor_sz_iterate(conf, + LorenzoPredictor(conf.absErrorBound), + quantizer, encoder, lossless); } else { predictors.push_back(std::make_shared>(conf.absErrorBound)); } } if (conf.lorenzo2) { if (use_single_predictor) { - return make_sz_general_compressor( - make_sz_general_frontend(conf, LorenzoPredictor(conf.absErrorBound), quantizer), - encoder, lossless); + return make_compressor_sz_iterate(conf, + LorenzoPredictor(conf.absErrorBound), + quantizer, encoder, lossless); } else { predictors.push_back(std::make_shared>(conf.absErrorBound)); } } if (conf.regression) { if (use_single_predictor) { - return make_sz_general_compressor( - make_sz_general_frontend(conf, RegressionPredictor(conf.blockSize, conf.absErrorBound), - quantizer), encoder, lossless); + return make_compressor_sz_iterate(conf, RegressionPredictor(conf.blockSize, conf.absErrorBound), + quantizer, encoder, lossless); } else { predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); } @@ -61,57 +60,54 @@ namespace SZ3 { if (conf.regression2) { if (use_single_predictor) { - return make_sz_general_compressor( - make_sz_general_frontend(conf, PolyRegressionPredictor(conf.blockSize, conf.absErrorBound), - quantizer), encoder, lossless); + return make_compressor_sz_iterate(conf, PolyRegressionPredictor(conf.blockSize, conf.absErrorBound), + quantizer, encoder, lossless); } else { predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); } } - return make_sz_general_compressor( - make_sz_general_frontend(conf, ComposedPredictor(predictors), - quantizer), encoder, lossless); + return make_compressor_sz_iterate(conf, ComposedPredictor(predictors), + quantizer, encoder, lossless); } template - char *SZ_compress_LorenzoReg(Config &conf, T *data, size_t &outSize) { + size_t SZ_compress_LorenzoReg(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { assert(N == conf.N); assert(conf.cmprAlgo == ALGO_LORENZO_REG); calAbsErrorBound(conf, data); - char *cmpData; auto quantizer = LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2); if (N == 3 && !conf.regression2 || (N == 1 && !conf.regression && !conf.regression2)) { // use fast version for 3D - auto sz = make_sz_general_compressor(make_sz_fast_frontend(conf, quantizer), HuffmanEncoder(), + auto sz = make_compressor_sz_generic(make_decomposition_lorenzo_regression(conf, quantizer), HuffmanEncoder(), Lossless_zstd()); - cmpData = (char *) sz->compress(conf, data, outSize); + return sz->compress(conf, data, cmpData, cmpCap); } else { - auto sz = make_lorenzo_regression_compressor(conf, quantizer, HuffmanEncoder(), Lossless_zstd()); - cmpData = (char *) sz->compress(conf, data, outSize); + auto sz = make_compressor_typetwo_lorenzo_regression(conf, quantizer, HuffmanEncoder(), Lossless_zstd()); + return sz->compress(conf, data, cmpData, cmpCap); } - return cmpData; +// return cmpData; } template - void SZ_decompress_LorenzoReg(const Config &conf, char *cmpData, size_t cmpSize, T *decData) { + void SZ_decompress_LorenzoReg(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { assert(conf.cmprAlgo == ALGO_LORENZO_REG); - uchar const *cmpDataPos = (uchar *) cmpData; + auto cmpDataPos = cmpData; LinearQuantizer quantizer; if (N == 3 && !conf.regression2 || (N == 1 && !conf.regression && !conf.regression2)) { // use fast version for 3D - auto sz = make_sz_general_compressor(make_sz_fast_frontend(conf, quantizer), + auto sz = make_compressor_sz_generic(make_decomposition_lorenzo_regression(conf, quantizer), HuffmanEncoder(), Lossless_zstd()); - sz->decompress(cmpDataPos, cmpSize, decData); + sz->decompress(conf, cmpDataPos, cmpSize, decData); return; } else { - auto sz = make_lorenzo_regression_compressor(conf, quantizer, HuffmanEncoder(), Lossless_zstd()); - sz->decompress(cmpDataPos, cmpSize, decData); + auto sz = make_compressor_typetwo_lorenzo_regression(conf, quantizer, HuffmanEncoder(), Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); return; } diff --git a/include/SZ3/api/impl/SZDispatcher.hpp b/include/SZ3/api/impl/SZDispatcher.hpp index e95ec4ca..94c0ef3b 100644 --- a/include/SZ3/api/impl/SZDispatcher.hpp +++ b/include/SZ3/api/impl/SZDispatcher.hpp @@ -4,47 +4,52 @@ #include "SZ3/utils/MemoryUtil.hpp" #include "SZ3/utils/Statistic.hpp" #include "SZ3/utils/Config.hpp" -#include "SZ3/api/impl/SZInterp.hpp" -#include "SZ3/api/impl/SZLorenzoReg.hpp" +#include "SZ3/api/impl/SZAlgoInterp.hpp" +#include "SZ3/api/impl/SZAlgoLorenzoReg.hpp" +#include "SZ3/api/impl/SZAlgo.hpp" #include namespace SZ3 { template - char *SZ_compress_dispatcher(Config &conf, T *data, size_t &outSize) { - + size_t SZ_compress_dispatcher(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); calAbsErrorBound(conf, data); - char *cmpData; +// char *cmpData; if (conf.absErrorBound == 0) { auto zstd = Lossless_zstd(); - cmpData = (char *) zstd.compress((uchar *) data, conf.num * sizeof(T), outSize); + return zstd.compress((uchar *) data, conf.num * sizeof(T), cmpData, cmpCap); } else if (conf.cmprAlgo == ALGO_LORENZO_REG) { - cmpData = (char *) SZ_compress_LorenzoReg(conf, data, outSize); + return SZ_compress_LorenzoReg(conf, data, cmpData, cmpCap); } else if (conf.cmprAlgo == ALGO_INTERP) { - cmpData = (char *) SZ_compress_Interp(conf, data, outSize); + return SZ_compress_Interp(conf, data, cmpData, cmpCap); } else if (conf.cmprAlgo == ALGO_INTERP_LORENZO) { - cmpData = (char *) SZ_compress_Interp_lorenzo(conf, data, outSize); + return SZ_compress_Interp_lorenzo(conf, data, cmpData, cmpCap); + } else if (conf.cmprAlgo == ALGO_NOPRED) { + return SZ_compress_nopred(conf, data, cmpData, cmpCap); } - return cmpData; + return 0; +// return cmpData; } - - + template - void SZ_decompress_dispatcher(Config &conf, char *cmpData, size_t cmpSize, T *decData) { + void SZ_decompress_dispatcher(Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { if (conf.absErrorBound == 0) { auto zstd = Lossless_zstd(); - auto zstdOut = zstd.decompress((uchar *) cmpData, cmpSize); - memcpy(decData, zstdOut, conf.num * sizeof(T)); + auto zstdDstCap = conf.num * sizeof(T); + zstd.decompress(cmpData, cmpSize, (uchar *) decData, zstdDstCap); } else if (conf.cmprAlgo == ALGO_LORENZO_REG) { SZ_decompress_LorenzoReg(conf, cmpData, cmpSize, decData); } else if (conf.cmprAlgo == ALGO_INTERP) { SZ_decompress_Interp(conf, cmpData, cmpSize, decData); + } else if (conf.cmprAlgo == ALGO_NOPRED) { + SZ_decompress_nopred(conf, cmpData, cmpSize, decData); } else { printf("SZ_decompress_dispatcher, Method not supported\n"); exit(0); } - + } } #endif \ No newline at end of file diff --git a/include/SZ3/api/impl/SZImpl.hpp b/include/SZ3/api/impl/SZImpl.hpp index 0ee35e2a..abbf52bd 100644 --- a/include/SZ3/api/impl/SZImpl.hpp +++ b/include/SZ3/api/impl/SZImpl.hpp @@ -8,22 +8,22 @@ namespace SZ3 { template - char *SZ_compress_impl(Config &conf, const T *data, size_t &outSize) { + size_t SZ_compress_impl(Config &conf, const T *data, uchar *cmpData, size_t cmpCap) { #ifndef _OPENMP conf.openmp=false; #endif if (conf.openmp) { //dataCopy for openMP is handled by each thread - return SZ_compress_OMP(conf, data, outSize); + return SZ_compress_OMP(conf, data, cmpData, cmpCap); } else { std::vector dataCopy(data, data + conf.num); - return SZ_compress_dispatcher(conf, dataCopy.data(), outSize); + return SZ_compress_dispatcher(conf, dataCopy.data(), cmpData, cmpCap); } } template - void SZ_decompress_impl(Config &conf, char *cmpData, size_t cmpSize, T *decData) { + void SZ_decompress_impl(Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { #ifndef _OPENMP diff --git a/include/SZ3/api/impl/SZImplOMP.hpp b/include/SZ3/api/impl/SZImplOMP.hpp index 868ac84e..eb9f4a4d 100644 --- a/include/SZ3/api/impl/SZImplOMP.hpp +++ b/include/SZ3/api/impl/SZImplOMP.hpp @@ -13,14 +13,14 @@ #endif namespace SZ3 { template - char *SZ_compress_OMP(Config &conf, const T *data, size_t &outSize) { - unsigned char *buffer, *buffer_pos; + size_t SZ_compress_OMP(Config &conf, const T *data, uchar *cmpData, size_t cmpCap) { + unsigned char *buffer_pos = cmpData; #ifdef _OPENMP assert(N == conf.N); - std::vector compressed_t; + std::vector compressed_t; std::vector cmp_size_t, cmp_start_t; std::vector min_t, max_t; std::vector conf_t; @@ -74,7 +74,9 @@ namespace SZ3 { conf_t[tid] = conf; conf_t[tid].setDims(dims_t.begin(), dims_t.end()); - compressed_t[tid] = SZ_compress_dispatcher(conf_t[tid], data_t.data(), cmp_size_t[tid]); + cmp_size_t[tid] = data_t.size() * sizeof(T); + compressed_t[tid] = (uchar *) malloc(cmp_size_t[tid]); + SZ_compress_dispatcher(conf_t[tid], data_t.data(), compressed_t[tid], cmp_size_t[tid]); #pragma omp barrier #pragma omp single @@ -86,8 +88,8 @@ namespace SZ3 { cmp_start_t[i] = cmp_start_t[i - 1] + cmp_size_t[i - 1]; } size_t bufferSize = sizeof(int) + (nThreads + 1) * Config::size_est() + cmp_start_t[nThreads]; - buffer = new uchar[bufferSize]; - buffer_pos = buffer; +// buffer = new uchar[bufferSize]; +// buffer_pos = buffer; write(nThreads, buffer_pos); for (int i = 0; i < nThreads; i++) { conf_t[i].save(buffer_pos); @@ -96,22 +98,23 @@ namespace SZ3 { } memcpy(buffer_pos + cmp_start_t[tid], compressed_t[tid], cmp_size_t[tid]); - delete[] compressed_t[tid]; + free(compressed_t[tid]); } - - outSize = buffer_pos - buffer + cmp_start_t[nThreads]; + + return buffer_pos - cmpData + cmp_start_t[nThreads]; // timer.stop("OMP memcpy"); #endif - return (char *) buffer; +// return (char *) buffer; + return 0; } template - void SZ_decompress_OMP(const Config &conf, char *cmpData, size_t cmpSize, T *decData) { + void SZ_decompress_OMP(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { #ifdef _OPENMP - const unsigned char *cmpr_data_pos = (unsigned char *) cmpData; + auto cmpr_data_pos = cmpData; int nThreads = 1; read(nThreads, cmpr_data_pos); omp_set_num_threads(nThreads); @@ -125,7 +128,7 @@ namespace SZ3 { std::vector cmp_start_t, cmp_size_t; cmp_size_t.resize(nThreads); read(cmp_size_t.data(), nThreads, cmpr_data_pos); - char *cmpr_data_p = cmpData + (cmpr_data_pos - (unsigned char *) cmpData); + auto cmpr_data_p = cmpr_data_pos; cmp_start_t.resize(nThreads + 1); cmp_start_t[0] = 0; diff --git a/include/SZ3/api/sz.hpp b/include/SZ3/api/sz.hpp index 3658a752..2239190b 100644 --- a/include/SZ3/api/sz.hpp +++ b/include/SZ3/api/sz.hpp @@ -1,7 +1,6 @@ #ifndef SZ3_SZ_HPP #define SZ3_SZ_HPP - #include "SZ3/api/impl/SZImpl.hpp" #include "SZ3/version.hpp" #include @@ -51,31 +50,47 @@ conf.absErrorBound = 1E-3; // absolute error bound 1e-3 char *compressedData = SZ_compress(conf, data, outSize); */ template -char *SZ_compress(const SZ3::Config &conf, const T *data, size_t &cmpSize) { +size_t SZ_compress(const SZ3::Config &conf_, const T *data, char *cmpData, size_t cmpCap) { using namespace SZ3; - Config confCopy(conf); - char *cmpData; + Config conf(conf_); + + if (cmpCap < conf.num * sizeof(T)) { + throw std::invalid_argument( + "cmpCap too small, remember to initialize the cmpCap with at least the same size of the original data"); + } + + auto dst = (uchar*)cmpData + conf.size_est(); + auto dstCap = cmpCap - conf.size_est(); + + size_t dstLen = 0; if (conf.N == 1) { - cmpData = SZ_compress_impl(confCopy, data, cmpSize); + dstLen = SZ_compress_impl(conf, data, dst, dstCap); } else if (conf.N == 2) { - cmpData = SZ_compress_impl(confCopy, data, cmpSize); + dstLen = SZ_compress_impl(conf, data, dst, dstCap); } else if (conf.N == 3) { - cmpData = SZ_compress_impl(confCopy, data, cmpSize); + dstLen = SZ_compress_impl(conf, data, dst, dstCap); } else if (conf.N == 4) { - cmpData = SZ_compress_impl(confCopy, data, cmpSize); + dstLen = SZ_compress_impl(conf, data, dst, dstCap); } else { printf("Data dimension higher than 4 is not supported.\n"); exit(0); } - { - //save config - uchar *cmpDataPos = (uchar *) cmpData + cmpSize; - confCopy.save(cmpDataPos); - size_t newSize = (char *) cmpDataPos - cmpData; - write(int(newSize - cmpSize), cmpDataPos); - cmpSize = (char *) cmpDataPos - cmpData; - } - return cmpData; + + + auto confPos = (uchar *) cmpData; + conf.save(confPos); + return conf.size_est() + dstLen; +} + +template +char *SZ_compress(const SZ3::Config &conf, const T *data, size_t &cmpSize) { + using namespace SZ3; + + size_t bufferLen = conf.num * sizeof(T) * 1.2; + auto buffer = new char[bufferLen]; + cmpSize = SZ_compress(conf, data, buffer, bufferLen); + + return buffer; } @@ -98,24 +113,28 @@ char *SZ_compress(const SZ3::Config &conf, const T *data, size_t &cmpSize) { template void SZ_decompress(SZ3::Config &conf, char *cmpData, size_t cmpSize, T *&decData) { using namespace SZ3; - { - //load config - int confSize; - memcpy(&confSize, cmpData + (cmpSize - sizeof(int)), sizeof(int)); - uchar const *cmpDataPos = (uchar *) cmpData + (cmpSize - sizeof(int) - confSize); - conf.load(cmpDataPos); + auto confPos = (const uchar *) cmpData; + conf.load(confPos); + if (conf.sz3MagicNumber != SZ3_MAGIC_NUMBER) { + throw std::invalid_argument("magic number mismatch, the input data is not compressed by SZ3"); + } + if (versionStr(conf.sz3DataVer) != SZ3_DATA_VER) { + std::stringstream ss; + printf("program v%s , program-data %s , input data v%s\n", SZ3_VER, SZ3_DATA_VER, versionStr(conf.sz3DataVer).data()); + ss << "Please use SZ3 v" << conf.sz3DataVer << " to decompress the data" << std::endl; + throw std::invalid_argument(ss.str()); } if (decData == nullptr) { decData = new T[conf.num]; } if (conf.N == 1) { - SZ_decompress_impl(conf, cmpData, cmpSize, decData); + SZ_decompress_impl(conf, confPos, cmpSize, decData); } else if (conf.N == 2) { - SZ_decompress_impl(conf, cmpData, cmpSize, decData); + SZ_decompress_impl(conf, confPos, cmpSize, decData); } else if (conf.N == 3) { - SZ_decompress_impl(conf, cmpData, cmpSize, decData); + SZ_decompress_impl(conf, confPos, cmpSize, decData); } else if (conf.N == 4) { - SZ_decompress_impl(conf, cmpData, cmpSize, decData); + SZ_decompress_impl(conf, confPos, cmpSize, decData); } else { printf("Data dimension higher than 4 is not supported.\n"); exit(0); diff --git a/include/SZ3/compressor/Compressor.hpp b/include/SZ3/compressor/Compressor.hpp index fd17fdc5..e663d694 100644 --- a/include/SZ3/compressor/Compressor.hpp +++ b/include/SZ3/compressor/Compressor.hpp @@ -17,33 +17,25 @@ namespace SZ3::concepts { class CompressorInterface { public: - /** - * decompress data to a new() space created in this function - * - * @param cmpData compressed data in bytes - * @param cmpSize size of compressed data in bytes - * @param num size of original data - * @return pointer to the decompress data - */ - virtual T *decompress(uchar const *cmpData, const size_t &cmpSize, size_t num) = 0; /** - * decompress data to a pre-allocated space + * decompress data * @param cmpData compressed data in bytes * @param cmpSize size of compressed data in bytes * @param decData pre-allocated space to store the decompress data * @return pointer to the decompress data */ - virtual T *decompress(uchar const *cmpData, const size_t &cmpSize, T *decData) = 0; + virtual T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) = 0; /** - * compress data to a new() space + * compress data * @param conf compression configuration * @param data input data in original format - * @param compressed_size size of the compressed data in bytes - * @return compressed data in bytes + * @param cmpData compressed data in bytes + * @param cmpCap size of the compressed buffer in bytes + * @return size of the compressed data in bytes */ - virtual uchar *compress(const Config &conf, T *data, size_t &compressed_size) = 0; + virtual size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) = 0; }; } diff --git a/include/SZ3/compressor/SZGeneralCompressor.hpp b/include/SZ3/compressor/SZGeneralCompressor.hpp deleted file mode 100644 index ed083b81..00000000 --- a/include/SZ3/compressor/SZGeneralCompressor.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef SZ_GENERAL_COMPRESSOR_HPP -#define SZ_GENERAL_COMPRESSOR_HPP - -#include "SZ3/compressor/Compressor.hpp" -#include "SZ3/frontend/Frontend.hpp" -#include "SZ3/encoder/Encoder.hpp" -#include "SZ3/lossless/Lossless.hpp" -#include "SZ3/utils/FileUtil.hpp" -#include "SZ3/utils/Config.hpp" -#include "SZ3/utils/Timer.hpp" -#include "SZ3/def.hpp" -#include - -namespace SZ3 { - template - class SZGeneralCompressor : public concepts::CompressorInterface { - public: - - - SZGeneralCompressor(Frontend frontend, Encoder encoder, Lossless lossless) : - frontend(frontend), encoder(encoder), lossless(lossless) { - static_assert(std::is_base_of, Frontend>::value, - "must implement the frontend interface"); - static_assert(std::is_base_of, Encoder>::value, - "must implement the encoder interface"); - static_assert(std::is_base_of::value, - "must implement the lossless interface"); - } - - uchar *compress(const Config &conf, T *data, size_t &compressed_size) { - - std::vector quant_inds = frontend.compress(data); - - encoder.preprocess_encode(quant_inds, frontend.get_radius() * 2); - size_t bufferSize = 1.2 * (frontend.size_est() + encoder.size_est() + sizeof(T) * quant_inds.size()); - - uchar *buffer = new uchar[bufferSize]; - uchar *buffer_pos = buffer; - - write(conf.num, buffer_pos); - - frontend.save(buffer_pos); - - encoder.save(buffer_pos); - encoder.encode(quant_inds, buffer_pos); - encoder.postprocess_encode(); - - assert(buffer_pos - buffer < bufferSize); - - uchar *lossless_data = lossless.compress(buffer, buffer_pos - buffer, compressed_size); - lossless.postcompress_data(buffer); - - return lossless_data; - } - - T *decompress(uchar const *cmpData, const size_t &cmpSize, size_t num) { - T *dec_data = new T[num]; - return decompress(cmpData, cmpSize, dec_data); - } - - T *decompress(uchar const *cmpData, const size_t &cmpSize, T *decData) { - size_t remaining_length = cmpSize; - -// Timer timer(true); - auto buffer = lossless.decompress(cmpData, remaining_length); - uchar const *buffer_pos = buffer; -// timer.stop("Lossless"); - size_t num = 0; - read(num, buffer_pos, remaining_length); - - frontend.load(buffer_pos, remaining_length); - - encoder.load(buffer_pos, remaining_length); - -// timer.start(); - auto quant_inds = encoder.decode(buffer_pos, num); - encoder.postprocess_decode(); -// timer.stop("Decoder"); - - lossless.postdecompress_data(buffer); - -// timer.start(); - frontend.decompress(quant_inds, decData); -// timer.stop("Prediction & Recover"); - return decData; - } - - - private: - Frontend frontend; - Encoder encoder; - Lossless lossless; - }; - - template - std::shared_ptr> - make_sz_general_compressor(Frontend frontend, Encoder encoder, Lossless lossless) { - return std::make_shared>(frontend, encoder, lossless); - } - - -} -#endif diff --git a/include/SZ3/compressor/SZGenericCompressor.hpp b/include/SZ3/compressor/SZGenericCompressor.hpp new file mode 100644 index 00000000..4d784a93 --- /dev/null +++ b/include/SZ3/compressor/SZGenericCompressor.hpp @@ -0,0 +1,92 @@ +#ifndef SZ_COMPRESSOR_TYPE_ONE_HPP +#define SZ_COMPRESSOR_TYPE_ONE_HPP + +#include "SZ3/compressor/Compressor.hpp" +#include "SZ3/decomposition/Decomposition.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Timer.hpp" +#include "SZ3/def.hpp" +#include + +/** + * SZGenericCompressor glues together decomposition, encoder, and lossless modules to form the compression pipeline + * it doesn't contains the logic to iterate through the input data. The logic is handled inside decomposition + */ + +namespace SZ3 { + template + class SZGenericCompressor : public concepts::CompressorInterface { + public: + + + SZGenericCompressor(Decomposition decomposition, Encoder encoder, Lossless lossless) : + decomposition(decomposition), encoder(encoder), lossless(lossless) { + static_assert(std::is_base_of, Decomposition>::value, + "must implement the frontend interface"); + static_assert(std::is_base_of, Encoder>::value, + "must implement the encoder interface"); + static_assert(std::is_base_of::value, + "must implement the lossless interface"); + } + + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + + std::vector quant_inds = decomposition.compress(conf, data); + + encoder.preprocess_encode(quant_inds, decomposition.get_radius() * 2); + size_t bufferSize = std::max(1000, 1.2 * (decomposition.size_est() + encoder.size_est() + sizeof(T) * quant_inds.size())); + + auto buffer = (uchar *) malloc(bufferSize); + uchar *buffer_pos = buffer; + + decomposition.save(buffer_pos); + + encoder.save(buffer_pos); + encoder.encode(quant_inds, buffer_pos); + encoder.postprocess_encode(); + + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + free(buffer); + + return cmpSize; + } + + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) { + size_t bufferCap = conf.num * sizeof(T); + auto buffer = (uchar *) malloc(bufferCap); + lossless.decompress(cmpData, cmpSize, buffer, bufferCap); + + size_t remaining_length = bufferCap; + uchar const *buffer_pos = buffer; + + decomposition.load(buffer_pos, remaining_length); + encoder.load(buffer_pos, remaining_length); + + auto quant_inds = encoder.decode(buffer_pos, conf.num); + encoder.postprocess_decode(); + + free(buffer); + + decomposition.decompress(conf, quant_inds, decData); + return decData; + } + + + private: + Decomposition decomposition; + Encoder encoder; + Lossless lossless; + }; + + template + std::shared_ptr> + make_compressor_sz_generic(Decomposition decomposition, Encoder encoder, Lossless lossless) { + return std::make_shared>(decomposition, encoder, lossless); + } + + +} +#endif diff --git a/include/SZ3/compressor/SZIterateCompressor.hpp b/include/SZ3/compressor/SZIterateCompressor.hpp new file mode 100644 index 00000000..81d046a3 --- /dev/null +++ b/include/SZ3/compressor/SZIterateCompressor.hpp @@ -0,0 +1,180 @@ +#ifndef SZ_COMPRESSOR_TYPE_TWO_HPP +#define SZ_COMPRESSOR_TYPE_TWO_HPP + +#include "SZ3/compressor/Compressor.hpp" +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/utils/Iterator.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Timer.hpp" +#include "SZ3/def.hpp" +#include + +/** + * SZIterateCompressor glues together predictor, quantizer, encoder, and lossless modules to form the compression pipeline + * it contains the logic to iterate through the input data + */ +namespace SZ3 { + template + class SZIterateCompressor : public concepts::CompressorInterface { + public: + + + SZIterateCompressor(const Config &conf, Predictor predictor, Quantizer quantizer, Encoder encoder, Lossless lossless) : + fallback_predictor(LorenzoPredictor(conf.absErrorBound)), + predictor(predictor), + quantizer(quantizer), + block_size(conf.blockSize), + num_elements(conf.num), encoder(encoder), lossless(lossless) { + std::copy_n(conf.dims.begin(), N, global_dimensions.begin()); + static_assert(std::is_base_of, Predictor>::value, + "must implement the predictor interface"); + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quantizer interface"); + static_assert(std::is_base_of, Encoder>::value, + "must implement the encoder interface"); + static_assert(std::is_base_of::value, + "must implement the lossless interface"); + } + + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + + std::vector quant_inds(num_elements); + auto block_range = std::make_shared>( + data, std::begin(global_dimensions), std::end(global_dimensions), block_size, 0); + + auto element_range = std::make_shared>( + data, std::begin(global_dimensions), std::end(global_dimensions), 1, 0); + + predictor.precompress_data(block_range->begin()); + quantizer.precompress_data(); + size_t quant_count = 0; + for (auto block = block_range->begin(); block != block_range->end(); ++block) { + + element_range->update_block_range(block, block_size); + + concepts::PredictorInterface *predictor_withfallback = &predictor; + if (!predictor.precompress_block(element_range)) { + predictor_withfallback = &fallback_predictor; + } + predictor_withfallback->precompress_block_commit(); + + for (auto element = element_range->begin(); element != element_range->end(); ++element) { + quant_inds[quant_count++] = quantizer.quantize_and_overwrite( + *element, predictor_withfallback->predict(element)); + } + } + + predictor.postcompress_data(block_range->begin()); + quantizer.postcompress_data(); + + encoder.preprocess_encode(quant_inds, quantizer.get_radius() * 2); + + size_t bufferSize = 1.2 * (quantizer.size_est() + encoder.size_est() + sizeof(T) * quant_inds.size()); + auto buffer = (uchar *) malloc(bufferSize); + uchar *buffer_pos = buffer; + + write(conf.num, buffer_pos); + write(global_dimensions.data(), N, buffer_pos); + write(block_size, buffer_pos); + + predictor.save(buffer_pos); + quantizer.save(buffer_pos); + + encoder.save(buffer_pos); + encoder.encode(quant_inds, buffer_pos); + encoder.postprocess_encode(); + + assert(buffer_pos - buffer < bufferSize); + + auto cmpSize=lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + free(buffer); + return cmpSize; + } + + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) { +// Timer timer(true); + size_t bufferCap = conf.num * sizeof(T); + auto buffer = (uchar *) malloc(bufferCap); + lossless.decompress(cmpData, cmpSize, buffer, bufferCap); + size_t remaining_length = bufferCap; + uchar const *buffer_pos = buffer; +// timer.stop("Lossless"); + size_t num = 0; + read(num, buffer_pos, remaining_length); + + read(global_dimensions.data(), N, buffer_pos, remaining_length); + num_elements = 1; + for (const auto &d: global_dimensions) { + num_elements *= d; + } + read(block_size, buffer_pos, remaining_length); + predictor.load(buffer_pos, remaining_length); + quantizer.load(buffer_pos, remaining_length); + + encoder.load(buffer_pos, remaining_length); + +// timer.start(); + auto quant_inds = encoder.decode(buffer_pos, num); + encoder.postprocess_decode(); +// timer.stop("Decoder"); + + free(buffer); +// lossless.postdecompress_data(buffer); + +// timer.start(); + int const *quant_inds_pos = (int const *) quant_inds.data(); + std::array intra_block_dims; + auto block_range = std::make_shared>( + decData, std::begin(global_dimensions), std::end(global_dimensions), block_size, 0); + + auto element_range = std::make_shared>( + decData, std::begin(global_dimensions), std::end(global_dimensions), 1, 0); + + predictor.predecompress_data(block_range->begin()); + quantizer.predecompress_data(); + + for (auto block = block_range->begin(); block != block_range->end(); ++block) { + + element_range->update_block_range(block, block_size); + + concepts::PredictorInterface *predictor_withfallback = &predictor; + if (!predictor.predecompress_block(element_range)) { + predictor_withfallback = &fallback_predictor; + } + for (auto element = element_range->begin(); element != element_range->end(); ++element) { + *element = quantizer.recover(predictor_withfallback->predict(element), *(quant_inds_pos++)); + } + } + predictor.postdecompress_data(block_range->begin()); + quantizer.postdecompress_data(); + return decData; +// timer.stop("Prediction & Recover"); + } + + + private: + Predictor predictor; + LorenzoPredictor fallback_predictor; + Quantizer quantizer; + uint block_size; + size_t num_elements; + std::array global_dimensions; + Encoder encoder; + Lossless lossless; + }; + + template + std::shared_ptr> + make_compressor_sz_iterate(const Config &conf, Predictor predictor, Quantizer quantizer, Encoder encoder, Lossless lossless) { + return std::make_shared>(conf, predictor, quantizer, encoder, lossless); + } + + +} +#endif diff --git a/include/SZ3/compressor/SZTruncateCompressor.hpp b/include/SZ3/compressor/SZTruncateCompressor.hpp deleted file mode 100644 index 6be5e260..00000000 --- a/include/SZ3/compressor/SZTruncateCompressor.hpp +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef SZ_Truncate_COMPRESSOR_HPP -#define SZ_Truncate_COMPRESSOR_HPP - -#include "SZ3/compressor/Compressor.hpp" -#include "SZ3/frontend/Frontend.hpp" -#include "SZ3/encoder/Encoder.hpp" -#include "SZ3/lossless/Lossless.hpp" -#include "SZ3/utils/FileUtil.hpp" -#include "SZ3/utils/Config.hpp" -#include "SZ3/utils/Timer.hpp" -#include "SZ3/utils/ByteUtil.hpp" -#include "SZ3/utils/MemoryUtil.hpp" -#include "SZ3/def.hpp" -#include - -namespace SZ3 { - template - class SZTruncateCompressor : public concepts::CompressorInterface { - public: - - - SZTruncateCompressor(const Config &conf, Lossless lossless, int byteLens) : - lossless(lossless), conf(conf), byteLen(byteLens) { - static_assert(std::is_base_of::value, - "must implement the lossless interface"); - } - - uchar *compress(const Config &conf, T *data, size_t &compressed_size) { - - auto compressed_data = new uchar[conf.num * sizeof(T)]; - auto compressed_data_pos = (uchar *) compressed_data; - - Timer timer(true); - truncateArray(data, conf.num, byteLen, compressed_data_pos); - timer.stop("Prediction & Quantization"); - - uchar *lossless_data = lossless.compress(compressed_data, - (uchar *) compressed_data_pos - compressed_data, - compressed_size); - lossless.postcompress_data(compressed_data); - return lossless_data; - } - - T *decompress(uchar const *cmpData, const size_t &cmpSize, size_t num) { - T *dec_data = new T[num]; - return decompress(cmpData, cmpSize, dec_data); - } - - T *decompress(uchar const *cmpData, const size_t& cmpSize, T *decData) { - size_t remaining_length = cmpSize; - - auto compressed_data = lossless.decompress(cmpData, remaining_length); - auto compressed_data_pos = (uchar *) compressed_data; - - Timer timer(true); -// auto dec_data = new T[conf.num]; - truncateArrayRecover(compressed_data_pos, conf.num, byteLen, decData); - - lossless.postdecompress_data(compressed_data); - timer.stop("Prediction & Recover"); - return decData; - } - - - - private: - Lossless lossless; - Config conf; - int byteLen = 2; - }; - - template - SZTruncateCompressor - make_sz_truncate_compressor(const Config &conf, Lossless lossless, int byteLens) { - return SZTruncateCompressor(conf, lossless, byteLens); - } -} -#endif diff --git a/include/SZ3/compressor/deprecated/SZBlockInterpolationCompressor.hpp b/include/SZ3/compressor/specialized/SZBlockInterpolationCompressor.hpp similarity index 95% rename from include/SZ3/compressor/deprecated/SZBlockInterpolationCompressor.hpp rename to include/SZ3/compressor/specialized/SZBlockInterpolationCompressor.hpp index 94965d24..6037692c 100644 --- a/include/SZ3/compressor/deprecated/SZBlockInterpolationCompressor.hpp +++ b/include/SZ3/compressor/specialized/SZBlockInterpolationCompressor.hpp @@ -14,6 +14,10 @@ #include #include +/** + * DO NOT use this one + * It will be replaced by QOZ + */ namespace SZ3 { template class SZBlockInterpolationCompressor { @@ -32,16 +36,13 @@ namespace SZ3 { } - T *decompress(uchar const *cmpData, const size_t &cmpSize, size_t num) { - T *dec_data = new T[num]; - return decompress(cmpData, cmpSize, dec_data); - } - - T *decompress(uchar const *cmpData, const size_t &cmpSize, T *decData) { - size_t remaining_length = cmpSize; + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) { - uchar *buffer = lossless.decompress(cmpData, remaining_length); + size_t bufferCap = conf.num * sizeof(T); + auto buffer = (uchar *) malloc(bufferCap); + lossless.decompress(cmpData, cmpSize, buffer, bufferCap); + size_t remaining_length = bufferCap; uchar const *buffer_pos = buffer; read(global_dimensions.data(), N, buffer_pos, remaining_length); @@ -59,13 +60,13 @@ namespace SZ3 { encoder.postprocess_decode(); - lossless.postdecompress_data(buffer); + free(buffer); auto range = std::make_shared>(decData, - std::begin(global_dimensions), - std::end(global_dimensions), - block_size, - 0); + std::begin(global_dimensions), + std::end(global_dimensions), + block_size, + 0); quantizer.predecompress_data(); @@ -106,7 +107,7 @@ namespace SZ3 { // compress given the error bound - uchar *compress(const Config &conf, T *data, size_t &compressed_size) { + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) { block_size = conf.blockSize; num_elements = conf.num; @@ -118,9 +119,9 @@ namespace SZ3 { quant_inds.clear(); auto range = std::make_shared>(data, - std::begin(global_dimensions), - std::end(global_dimensions), - block_size, 0); + std::begin(global_dimensions), + std::end(global_dimensions), + block_size, 0); quantizer.precompress_data(); for (auto block = range->begin(); block != range->end(); ++block) { @@ -149,7 +150,7 @@ namespace SZ3 { encoder.preprocess_encode(quant_inds, quantizer.get_radius() * 2); size_t bufferSize = 1.2 * (quantizer.size_est() + encoder.size_est() + sizeof(T) * quant_inds.size()); - uchar *buffer = new uchar[bufferSize]; + auto *buffer = (uchar *) malloc(bufferSize); uchar *buffer_pos = buffer; write(global_dimensions.data(), N, buffer_pos); @@ -164,13 +165,13 @@ namespace SZ3 { encoder.postprocess_encode(); assert(buffer_pos - buffer < bufferSize); + + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + free(buffer); + return cmpSize; +// lossless.postcompress_data(buffer); - uchar *lossless_data = lossless.compress(buffer, - buffer_pos - buffer, - compressed_size); - lossless.postcompress_data(buffer); - - return lossless_data; +// return lossless_data; } private: diff --git a/tools/mdz/include/ExaaltCompressor.hpp b/include/SZ3/compressor/specialized/SZExaaltCompressor.hpp similarity index 50% rename from tools/mdz/include/ExaaltCompressor.hpp rename to include/SZ3/compressor/specialized/SZExaaltCompressor.hpp index bd36535c..98e16946 100644 --- a/tools/mdz/include/ExaaltCompressor.hpp +++ b/include/SZ3/compressor/specialized/SZExaaltCompressor.hpp @@ -1,60 +1,46 @@ #ifndef _SZ_EXAALT_COMPRESSSOR_HPP #define _SZ_EXAALT_COMPRESSSOR_HPP -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/utils/Iterator.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/def.hpp" + +/** + * specialized compressor for solid-state molecular dynamics simulation data + * it utilizes the data character of levels to boost compression ratio + * see ICDE'22 MDZ paper for details + */ namespace SZ3 { template - class SZ_Exaalt_Compressor { + class SZExaaltCompressor : public SZ3::concepts::CompressorInterface { public: - SZ_Exaalt_Compressor(const Config &conf, - Quantizer quantizer, - Encoder encoder, - Lossless lossless, - int timestep_op) : - quantizer(quantizer), encoder(encoder), lossless(lossless), - num_elements(conf.num), timestep_op(timestep_op) { + SZExaaltCompressor(Quantizer quantizer, Encoder encoder, Lossless lossless, int timestep_op) : + quantizer(quantizer), encoder(encoder), lossless(lossless), timestep_op(timestep_op) { static_assert(std::is_base_of, Quantizer>::value, "must implement the quatizer interface"); static_assert(std::is_base_of, Encoder>::value, "must implement the encoder interface"); static_assert(std::is_base_of::value, "must implement the lossless interface"); - assert(!(timestep_op > 0 && conf.dims.size() != 2) && - "timestep prediction requires 2d dataset"); - global_dimensions[0] = conf.dims[0]; - global_dimensions[1] = conf.dims[1]; - } - - void set_level(float level_start_, float level_offset_, int level_num_) { - this->level_start = level_start_; - this->level_offset = level_offset_; - this->level_num = level_num_ + 200; - } - - inline int quantize_to_level(T data) { - return round((data - level_start) / level_offset); } - inline T level(int l) { - return level_start + l * level_offset; - } // compress given the error bound - uchar *compress(T *data, size_t &compressed_size) { +// uchar *compress(T *data, size_t &compressed_size) { + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(!(timestep_op > 0 && conf.dims.size() != 2) && + "timestep prediction requires 2d dataset"); - std::vector quant_inds(num_elements); - std::vector pred_inds(num_elements); + std::vector quant_inds(conf.num); + std::vector pred_inds(conf.num); quantizer.precompress_data(); // Timer timer(true); @@ -64,7 +50,7 @@ namespace SZ3 { quant_inds[0] = quantizer.quantize_and_overwrite(data[0], level(l0)); if (timestep_op == 0) { - for (size_t i = 1; i < num_elements; i++) { + for (size_t i = 1; i < conf.num; i++) { auto l = quantize_to_level(data[i]); pred_inds[i] = l - l0 + level_num; quant_inds[i] = quantizer.quantize_and_overwrite(data[i], level(l)); @@ -72,28 +58,28 @@ namespace SZ3 { } } else { - std::vector levels(global_dimensions[1]); + std::vector levels(conf.dims[1]); levels[0] = l0; - for (size_t i = 1; i < global_dimensions[1]; i++) { + for (size_t i = 1; i < conf.dims[1]; i++) { levels[i] = quantize_to_level(data[i]); pred_inds[i] = levels[i] - levels[i - 1] + level_num; quant_inds[i] = quantizer.quantize_and_overwrite(data[i], level(levels[i])); } - auto pred_idx = global_dimensions[1]; + auto pred_idx = conf.dims[1]; if (timestep_op == 1) { - for (size_t i = 0; i < global_dimensions[1]; i++) { - for (size_t t = 1; t < global_dimensions[0]; t++) { - size_t idx = t * global_dimensions[1] + i; + for (size_t i = 0; i < conf.dims[1]; i++) { + for (size_t t = 1; t < conf.dims[0]; t++) { + size_t idx = t * conf.dims[1] + i; quant_inds[pred_idx++] = quantizer.quantize_and_overwrite( - data[idx], data[(t - 1) * global_dimensions[1] + i]); + data[idx], data[(t - 1) * conf.dims[1] + i]); } } - pred_inds.resize(global_dimensions[1]); + pred_inds.resize(conf.dims[1]); } else { - for (size_t i = 0; i < global_dimensions[1]; i++) { + for (size_t i = 0; i < conf.dims[1]; i++) { l0 = levels[i]; - for (size_t t = 1; t < global_dimensions[0]; t++) { - size_t idx = t * global_dimensions[1] + i; + for (size_t t = 1; t < conf.dims[0]; t++) { + size_t idx = t * conf.dims[1] + i; auto l = quantize_to_level(data[idx]); pred_inds[pred_idx] = l - l0 + level_num; quant_inds[pred_idx++] = quantizer.quantize_and_overwrite( @@ -102,63 +88,56 @@ namespace SZ3 { } } } - assert(pred_idx == num_elements); + assert(pred_idx == conf.num); } // timer.stop("Predition & Quantization"); quantizer.postcompress_data(); - uchar *compressed_data; - compressed_data = new uchar[4 * num_elements * sizeof(T)]; - uchar *compressed_data_pos = compressed_data; - write(global_dimensions.data(), N, compressed_data_pos); - quantizer.save(compressed_data_pos); + auto buffer = (uchar *) malloc(4 * conf.num * sizeof(T)); + uchar *buffer_pos = buffer; + quantizer.save(buffer_pos); // quantizer.print(); encoder.preprocess_encode(quant_inds, 2 * quantizer.get_radius()); - encoder.save(compressed_data_pos); - encoder.encode(quant_inds, compressed_data_pos); + encoder.save(buffer_pos); + encoder.encode(quant_inds, buffer_pos); encoder.postprocess_encode(); // std::cout << *std::min_element(pred_inds.begin(), pred_inds.end()) << std::endl; // std::cout << *std::max_element(pred_inds.begin(), pred_inds.end()) << std::endl; encoder.preprocess_encode(pred_inds, level_num * 2 + 1); - encoder.save(compressed_data_pos); - encoder.encode(pred_inds, compressed_data_pos); + encoder.save(buffer_pos); + encoder.encode(pred_inds, buffer_pos); encoder.postprocess_encode(); - - uchar *lossless_data = lossless.compress(compressed_data, compressed_data_pos - compressed_data, - compressed_size); - lossless.postcompress_data(compressed_data); - return lossless_data; + + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + free(buffer); + return cmpSize; } - T *decompress(uchar const *lossless_compressed_data, const size_t length) { - size_t remaining_length = length; - auto compressed_data = lossless.decompress(lossless_compressed_data, remaining_length); - uchar const *compressed_data_pos = compressed_data; - read(global_dimensions.data(), N, compressed_data_pos, remaining_length); - num_elements = 1; - for (const auto &d: global_dimensions) { - num_elements *= d; -// std::cout << d << " "; - } -// std::cout << std::endl; - quantizer.load(compressed_data_pos, remaining_length); - encoder.load(compressed_data_pos, remaining_length); - auto quant_inds = encoder.decode(compressed_data_pos, num_elements); - encoder.postprocess_decode(); +// T *decompress(uchar const *lossless_compressed_data, const size_t length) { + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *dec_data) { + + size_t bufferCap = conf.num * sizeof(T); + auto buffer = (uchar *) malloc(bufferCap); + lossless.decompress(cmpData, cmpSize, buffer, bufferCap); + size_t remaining_length = cmpSize; + uchar const *buffer_pos = buffer; - encoder.load(compressed_data_pos, remaining_length); - auto pred_inds_num = (timestep_op == 1) ? global_dimensions[1] : num_elements; - auto pred_inds = encoder.decode(compressed_data_pos, pred_inds_num); + quantizer.load(buffer_pos, remaining_length); + encoder.load(buffer_pos, remaining_length); + auto quant_inds = encoder.decode(buffer_pos, conf.num); encoder.postprocess_decode(); - lossless.postdecompress_data(compressed_data); + encoder.load(buffer_pos, remaining_length); + auto pred_inds_num = (timestep_op == 1) ? conf.dims[1] : conf.num; + auto pred_inds = encoder.decode(buffer_pos, pred_inds_num); + encoder.postprocess_decode(); - auto dec_data = new T[num_elements]; + free(buffer); quantizer.predecompress_data(); @@ -166,39 +145,39 @@ namespace SZ3 { dec_data[0] = quantizer.recover(level(l), quant_inds[0]); if (timestep_op == 0) { - for (size_t i = 1; i < num_elements; i++) { + for (size_t i = 1; i < conf.num; i++) { l += pred_inds[i] - level_num; dec_data[i] = quantizer.recover(level(l), quant_inds[i]); } } else { - std::vector levels(global_dimensions[1]); + std::vector levels(conf.dims[1]); levels[0] = l; - for (size_t i = 1; i < global_dimensions[1]; i++) { + for (size_t i = 1; i < conf.dims[1]; i++) { l += pred_inds[i] - level_num; dec_data[i] = quantizer.recover(level(l), quant_inds[i]); levels[i] = l; } - auto pred_idx = global_dimensions[1]; + auto pred_idx = conf.dims[1]; if (timestep_op == 1) { - for (size_t i = 0; i < global_dimensions[1]; i++) { - for (size_t t = 1; t < global_dimensions[0]; t++) { - dec_data[t * global_dimensions[1] + i] = + for (size_t i = 0; i < conf.dims[1]; i++) { + for (size_t t = 1; t < conf.dims[0]; t++) { + dec_data[t * conf.dims[1] + i] = quantizer.recover( - dec_data[(t - 1) * global_dimensions[1] + i], + dec_data[(t - 1) * conf.dims[1] + i], quant_inds[pred_idx++]); } } } else { - for (size_t i = 0; i < global_dimensions[1]; i++) { + for (size_t i = 0; i < conf.dims[1]; i++) { l = levels[i]; - for (size_t t = 1; t < global_dimensions[0]; t++) { - size_t idx = t * global_dimensions[1] + i; + for (size_t t = 1; t < conf.dims[0]; t++) { + size_t idx = t * conf.dims[1] + i; l += pred_inds[pred_idx] - level_num; dec_data[idx] = quantizer.recover(level(l), quant_inds[pred_idx++]); } } } - assert(pred_idx == num_elements); + assert(pred_idx == conf.num); } @@ -207,20 +186,37 @@ namespace SZ3 { return dec_data; } + void set_level(float level_start_, float level_offset_, int level_num_) { + this->level_start = level_start_; + this->level_offset = level_offset_; + this->level_num = level_num_ + 200; + } + + inline int quantize_to_level(T data) { + return round((data - level_start) / level_offset); + } + + inline T level(int l) { + return level_start + l * level_offset; + } + private: + Quantizer quantizer; Encoder encoder; Lossless lossless; - size_t num_elements; - std::array global_dimensions; - float level_start; - float level_offset; - int level_num; + float level_start{}; + float level_offset{}; + int level_num{}; int timestep_op; }; - + template + std::shared_ptr> + make_compressor_exaalt(Quantizer quantizer, Encoder encoder, Lossless lossless, int timestep_op) { + return std::make_shared>(quantizer, encoder, lossless, timestep_op); + } } #endif diff --git a/include/SZ3/compressor/specialized/SZTruncateCompressor.hpp b/include/SZ3/compressor/specialized/SZTruncateCompressor.hpp new file mode 100644 index 00000000..31f0756c --- /dev/null +++ b/include/SZ3/compressor/specialized/SZTruncateCompressor.hpp @@ -0,0 +1,77 @@ +#ifndef SZ_Truncate_COMPRESSOR_HPP +#define SZ_Truncate_COMPRESSOR_HPP + +#include "SZ3/compressor/Compressor.hpp" +#include "SZ3/decompositionTypeOne/Decomposition.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Timer.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "SZ3/def.hpp" +#include + +/** + * TODO + * merge this to SZCompressorTypeOne + * add truncate as typeOneDecomposition + * add bypass Encode + */ +namespace SZ3 { + template + class SZTruncateCompressor : public concepts::CompressorInterface { + public: + + SZTruncateCompressor(const Config &conf, Lossless lossless, int byteLens) : + lossless(lossless), conf(conf), byteLen(byteLens) { + static_assert(std::is_base_of::value, + "must implement the lossless interface"); + } + + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + + auto buffer = (uchar *) malloc(conf.num * sizeof(T)); + auto buffer_pos = (uchar *) buffer; + +// Timer timer(true); + truncateArray(data, conf.num, byteLen, buffer_pos); +// timer.stop("Prediction & Quantization"); + + auto cmpSize = lossless.compress(buffer, (uchar *) buffer_pos - buffer, cmpData, cmpCap); + free(buffer); + return cmpSize; +// lossless.postcompress_data(buffer); +// return lossless_data; + } + + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) { + size_t bufferCap = conf.num * sizeof(T); + auto buffer = (uchar *) malloc(bufferCap); + lossless.decompress(cmpData, cmpSize, buffer, bufferCap); + size_t remaining_length = bufferCap; + uchar const *buffer_pos = buffer; + +// Timer timer(true); +// auto dec_data = new T[conf.num]; + truncateArrayRecover(buffer_pos, conf.num, byteLen, decData); + + lossless.postdecompress_data(buffer); +// timer.stop("Prediction & Recover"); + return decData; + } + + private: + Lossless lossless; + Config conf; + int byteLen = 2; + }; + + template + SZTruncateCompressor + make_sz_truncate_compressor(const Config &conf, Lossless lossless, int byteLens) { + return SZTruncateCompressor(conf, lossless, byteLens); + } +} +#endif diff --git a/include/SZ3/frontend/Frontend.hpp b/include/SZ3/decomposition/Decomposition.hpp similarity index 58% rename from include/SZ3/frontend/Frontend.hpp rename to include/SZ3/decomposition/Decomposition.hpp index 3246ae75..d478741d 100644 --- a/include/SZ3/frontend/Frontend.hpp +++ b/include/SZ3/decomposition/Decomposition.hpp @@ -1,5 +1,5 @@ -#ifndef SZ3_FRONTEND_INTERFACE -#define SZ3_FRONTEND_INTERFACE +#ifndef SZ3_DECOMPOSITION_INTERFACE +#define SZ3_DECOMPOSITION_INTERFACE #include "SZ3/def.hpp" #include @@ -8,32 +8,34 @@ namespace SZ3::concepts { /** - * Frontend is the combination of predictor and quantizer - * It handles original data <--> quantized error (usually in integer) + * Decomposition defines transformation and prediction methods + * Transformation: + * original data <--> data transformed in another domain + * Prediction: + * original data <--> quantized prediction error + * combination of predictor (implementation) and quantizer (function calls) + * difference between Prediction and Decomposition interfaces: + * Prediction: + * prediction function takes scalar value (e.g, single data point) + * Decomposition: + * prediction function takes multidimensional tensors (e.g, whole input) * - * This will be changed to DecompositionTypeII - * DecompositionTypeI: - * E.g: transformation, predictors with no quantizer fused - * Example: regression, zfp, mgard, wavelet (cdf97 in sperr) - * DecompositionTypeII: - * predictors which internally calls quantizer APIs. - * example: Lorenzo, interpolation - * default quantizer need to be provided to reduce the initialization complexity for users * @tparam T original data * @tparam N data dimension */ template - class FrontendInterface { + class DecompositionInterface { public: - virtual ~FrontendInterface() = default; + virtual ~DecompositionInterface() = default; /** + * TODO allow T as output instead of int * predict the data and quantize the error * @param data original input * @return quantized prediction error */ - virtual std::vector compress(T *data) = 0; + virtual std::vector compress(const Config &conf, T *data) = 0; /** * reverse of compress(), reconstruct the data @@ -41,7 +43,7 @@ namespace SZ3::concepts { * @param dec_data place to write the reconstructed data * @return same value with dec_data */ - virtual T *decompress(std::vector &quant_inds, T *dec_data) = 0; + virtual T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) = 0; /** * serialize the frontend and store it to a buffer @@ -63,7 +65,7 @@ namespace SZ3::concepts { virtual void print() {}; - virtual void clear() {}; +// virtual void clear() {}; }; diff --git a/include/SZ3/compressor/SZInterpolationCompressor.hpp b/include/SZ3/decomposition/InterpolationDecomposition.hpp similarity index 74% rename from include/SZ3/compressor/SZInterpolationCompressor.hpp rename to include/SZ3/decomposition/InterpolationDecomposition.hpp index 3e92ab87..59a0e53d 100644 --- a/include/SZ3/compressor/SZInterpolationCompressor.hpp +++ b/include/SZ3/decomposition/InterpolationDecomposition.hpp @@ -1,11 +1,8 @@ -#ifndef _SZ_INTERPOLATION_COMPRESSSOR_HPP -#define _SZ_INTERPOLATION_COMPRESSSOR_HPP +#ifndef _SZ_INTERPOLATION_DECOMPOSITION_HPP +#define _SZ_INTERPOLATION_DECOMPOSITION_HPP -#include "SZ3/predictor/Predictor.hpp" -#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "Decomposition.hpp" #include "SZ3/quantizer/Quantizer.hpp" -#include "SZ3/encoder/Encoder.hpp" -#include "SZ3/lossless/Lossless.hpp" #include "SZ3/utils/Iterator.hpp" #include "SZ3/utils/MemoryUtil.hpp" #include "SZ3/utils/Config.hpp" @@ -18,50 +15,24 @@ #include namespace SZ3 { - template - class SZInterpolationCompressor { - public: - - - SZInterpolationCompressor(Quantizer quantizer, Encoder encoder, Lossless lossless) : - quantizer(quantizer), encoder(encoder), lossless(lossless) { - + template + class InterpolationDecomposition : public concepts::DecompositionInterface { + public: + + InterpolationDecomposition(const Config &conf, Quantizer quantizer) : quantizer(quantizer) { static_assert(std::is_base_of, Quantizer>::value, "must implement the quatizer interface"); - static_assert(std::is_base_of, Encoder>::value, - "must implement the encoder interface"); - static_assert(std::is_base_of::value, - "must implement the lossless interface"); - } - - T *decompress(uchar const *cmpData, const size_t &cmpSize, size_t num) { - T *dec_data = new T[num]; - return decompress(cmpData, cmpSize, dec_data); } - - T *decompress(uchar const *cmpData, const size_t &cmpSize, T *decData) { - size_t remaining_length = cmpSize; - uchar *buffer = lossless.decompress(cmpData, remaining_length); - uchar const *buffer_pos = buffer; - - read(global_dimensions.data(), N, buffer_pos, remaining_length); - read(blocksize, buffer_pos, remaining_length); - read(interpolator_id, buffer_pos, remaining_length); - read(direction_sequence_id, buffer_pos, remaining_length); - + + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) { init(); - - quantizer.load(buffer_pos, remaining_length); - encoder.load(buffer_pos, remaining_length); - quant_inds = encoder.decode(buffer_pos, num_elements); - - encoder.postprocess_decode(); - - lossless.postdecompress_data(buffer); + + this->quant_inds = quant_inds.data(); +// lossless.postdecompress_data(buffer); double eb = quantizer.get_eb(); - - *decData = quantizer.recover(0, quant_inds[quant_index++]); - + + *dec_data = quantizer.recover(0, this->quant_inds[quant_index++]); + for (uint level = interpolation_level; level > 0 && level <= interpolation_level; level--) { if (level >= 3) { quantizer.set_eb(eb * eb_ratio); @@ -70,9 +41,9 @@ namespace SZ3 { } size_t stride = 1U << (level - 1); auto inter_block_range = std::make_shared< - multi_dimensional_range>(decData, - std::begin(global_dimensions), std::end(global_dimensions), - stride * blocksize, 0); + multi_dimensional_range>(dec_data, + std::begin(global_dimensions), std::end(global_dimensions), + stride * blocksize, 0); auto inter_begin = inter_block_range->begin(); auto inter_end = inter_block_range->end(); for (auto block = inter_begin; block != inter_end; ++block) { @@ -83,36 +54,37 @@ namespace SZ3 { end_idx[i] = global_dimensions[i] - 1; } } - block_interpolation(decData, block.get_global_index(), end_idx, PB_recover, + block_interpolation(dec_data, block.get_global_index(), end_idx, PB_recover, interpolators[interpolator_id], direction_sequence_id, stride); - + } } quantizer.postdecompress_data(); // timer.stop("Interpolation Decompress"); - - return decData; + + return dec_data; } - + // compress given the error bound - uchar *compress(const Config &conf, T *data, size_t &compressed_size) { + std::vector compress(const Config &conf, T *data) { std::copy_n(conf.dims.begin(), N, global_dimensions.begin()); blocksize = conf.interpBlockSize; interpolator_id = conf.interpAlgo; direction_sequence_id = conf.interpDirection; - + init(); - quant_inds.reserve(num_elements); - size_t interp_compressed_size = 0; - + std::vector quant_inds_vec(num_elements); + quant_inds = quant_inds_vec.data(); + double eb = quantizer.get_eb(); - quant_inds.push_back(quantizer.quantize_and_overwrite(*data, 0)); - - Timer timer; - timer.start(); +// quant_inds.push_back(quantizer.quantize_and_overwrite(*data, 0)); + quant_inds[quant_index++] = quantizer.quantize_and_overwrite(*data, 0); +// Timer timer; +// timer.start(); + for (uint level = interpolation_level; level > 0 && level <= interpolation_level; level--) { if (level >= 3) { quantizer.set_eb(eb * eb_ratio); @@ -120,15 +92,15 @@ namespace SZ3 { quantizer.set_eb(eb); } size_t stride = 1U << (level - 1); - + auto inter_block_range = std::make_shared< - multi_dimensional_range>(data, std::begin(global_dimensions), - std::end(global_dimensions), - blocksize * stride, 0); - + multi_dimensional_range>(data, std::begin(global_dimensions), + std::end(global_dimensions), + blocksize * stride, 0); + auto inter_begin = inter_block_range->begin(); auto inter_end = inter_block_range->end(); - + for (auto block = inter_begin; block != inter_end; ++block) { auto end_idx = block.get_global_index(); for (int i = 0; i < N; i++) { @@ -137,55 +109,42 @@ namespace SZ3 { end_idx[i] = global_dimensions[i] - 1; } } - + block_interpolation(data, block.get_global_index(), end_idx, PB_predict_overwrite, interpolators[interpolator_id], direction_sequence_id, stride); } } - assert(quant_inds.size() == num_elements); -// timer.stop("Prediction & Quantization"); - -// writefile("pred.dat", preds.data(), num_elements); -// writefile("quant.dat", quant_inds.data(), num_elements); - encoder.preprocess_encode(quant_inds, quantizer.get_radius() * 2); - auto bufferSize = (size_t) std::max(1000.0, 1.2 * (quantizer.size_est() + encoder.size_est() + sizeof(T) * quant_inds.size())); - - uchar *buffer = new uchar[bufferSize]; - uchar *buffer_pos = buffer; - - write(global_dimensions.data(), N, buffer_pos); - write(blocksize, buffer_pos); - write(interpolator_id, buffer_pos); - write(direction_sequence_id, buffer_pos); - - quantizer.save(buffer_pos); + quantizer.postcompress_data(); - - timer.start(); - encoder.save(buffer_pos); - encoder.encode(quant_inds, buffer_pos); - encoder.postprocess_encode(); -// timer.stop("Coding"); - assert(buffer_pos - buffer < bufferSize); - - timer.start(); - uchar *lossless_data = lossless.compress(buffer, - buffer_pos - buffer, - compressed_size); - lossless.postcompress_data(buffer); -// timer.stop("Lossless"); - - compressed_size += interp_compressed_size; - return lossless_data; + return quant_inds_vec; } - - private: - + + void save(uchar *&c) { + write(global_dimensions.data(), N, c); + write(blocksize, c); + write(interpolator_id, c); + write(direction_sequence_id, c); + + quantizer.save(c); + } + + void load(const uchar *&c, size_t &remaining_length) { + read(global_dimensions.data(), N, c, remaining_length); + read(blocksize, c, remaining_length); + read(interpolator_id, c, remaining_length); + read(direction_sequence_id, c, remaining_length); + + quantizer.load(c, remaining_length); + } + + private: + enum PredictorBehavior { PB_predict_overwrite, PB_predict, PB_recover }; - + void init() { + quant_index = 0; assert(blocksize % 2 == 0 && "Interpolation block size should be even numbers"); num_elements = 1; interpolation_level = -1; @@ -195,12 +154,12 @@ namespace SZ3 { } num_elements *= global_dimensions[i]; } - + dimension_offsets[N - 1] = 1; for (int i = N - 2; i >= 0; i--) { dimension_offsets[i] = dimension_offsets[i + 1] * global_dimensions[i + 1]; } - + dimension_sequences = std::vector>(); auto sequence = std::array(); for (int i = 0; i < N; i++) { @@ -210,16 +169,15 @@ namespace SZ3 { dimension_sequences.push_back(sequence); } while (std::next_permutation(sequence.begin(), sequence.end())); } - + inline void quantize(size_t idx, T &d, T pred) { - quant_inds.push_back(quantizer.quantize_and_overwrite(d, pred)); + quant_inds[quant_index++] = (quantizer.quantize_and_overwrite(d, pred)); } - + inline void recover(size_t idx, T &d, T pred) { d = quantizer.recover(pred, quant_inds[quant_index++]); }; - - + double block_interpolation_1d(T *data, size_t begin, size_t end, size_t stride, const std::string &interp_func, const PredictorBehavior pb) { @@ -228,7 +186,7 @@ namespace SZ3 { return 0; } double predict_error = 0; - + size_t stride3x = 3 * stride; size_t stride5x = 5 * stride; if (interp_func == "linear" || n < 5) { @@ -261,7 +219,7 @@ namespace SZ3 { } } else { if (pb == PB_predict_overwrite) { - + T *d; size_t i; for (i = 3; i + 3 < n; i += 2) { @@ -271,46 +229,46 @@ namespace SZ3 { } d = data + begin + stride; quantize(d - data, *d, interp_quad_1(*(d - stride), *(d + stride), *(d + stride3x))); - + d = data + begin + i * stride; quantize(d - data, *d, interp_quad_2(*(d - stride3x), *(d - stride), *(d + stride))); if (n % 2 == 0) { d = data + begin + (n - 1) * stride; quantize(d - data, *d, interp_quad_3(*(d - stride5x), *(d - stride3x), *(d - stride))); } - + } else { T *d; - + size_t i; for (i = 3; i + 3 < n; i += 2) { d = data + begin + i * stride; recover(d - data, *d, interp_cubic(*(d - stride3x), *(d - stride), *(d + stride), *(d + stride3x))); } d = data + begin + stride; - + recover(d - data, *d, interp_quad_1(*(d - stride), *(d + stride), *(d + stride3x))); - + d = data + begin + i * stride; recover(d - data, *d, interp_quad_2(*(d - stride3x), *(d - stride), *(d + stride))); - + if (n % 2 == 0) { d = data + begin + (n - 1) * stride; recover(d - data, *d, interp_quad_3(*(d - stride5x), *(d - stride3x), *(d - stride))); } } } - + return predict_error; } - + template typename std::enable_if::type block_interpolation(T *data, std::array begin, std::array end, const PredictorBehavior pb, const std::string &interp_func, const int direction, size_t stride = 1) { return block_interpolation_1d(data, begin[0], end[0], stride, interp_func, pb); } - + template typename std::enable_if::type block_interpolation(T *data, std::array begin, std::array end, const PredictorBehavior pb, @@ -322,19 +280,19 @@ namespace SZ3 { size_t begin_offset = begin[dims[0]] * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[0]] - begin[dims[0]]) * dimension_offsets[dims[0]], + (end[dims[0]] - begin[dims[0]]) * dimension_offsets[dims[0]], stride * dimension_offsets[dims[0]], interp_func, pb); } for (size_t i = (begin[dims[0]] ? begin[dims[0]] + stride : 0); i <= end[dims[0]]; i += stride) { size_t begin_offset = i * dimension_offsets[dims[0]] + begin[dims[1]] * dimension_offsets[dims[1]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[1]] - begin[dims[1]]) * dimension_offsets[dims[1]], + (end[dims[1]] - begin[dims[1]]) * dimension_offsets[dims[1]], stride * dimension_offsets[dims[1]], interp_func, pb); } return predict_error; } - + template typename std::enable_if::type block_interpolation(T *data, std::array begin, std::array end, const PredictorBehavior pb, @@ -345,40 +303,39 @@ namespace SZ3 { for (size_t j = (begin[dims[1]] ? begin[dims[1]] + stride2x : 0); j <= end[dims[1]]; j += stride2x) { for (size_t k = (begin[dims[2]] ? begin[dims[2]] + stride2x : 0); k <= end[dims[2]]; k += stride2x) { size_t begin_offset = begin[dims[0]] * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + - k * dimension_offsets[dims[2]]; + k * dimension_offsets[dims[2]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[0]] - begin[dims[0]]) * - dimension_offsets[dims[0]], + (end[dims[0]] - begin[dims[0]]) * + dimension_offsets[dims[0]], stride * dimension_offsets[dims[0]], interp_func, pb); } } for (size_t i = (begin[dims[0]] ? begin[dims[0]] + stride : 0); i <= end[dims[0]]; i += stride) { for (size_t k = (begin[dims[2]] ? begin[dims[2]] + stride2x : 0); k <= end[dims[2]]; k += stride2x) { size_t begin_offset = i * dimension_offsets[dims[0]] + begin[dims[1]] * dimension_offsets[dims[1]] + - k * dimension_offsets[dims[2]]; + k * dimension_offsets[dims[2]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[1]] - begin[dims[1]]) * - dimension_offsets[dims[1]], + (end[dims[1]] - begin[dims[1]]) * + dimension_offsets[dims[1]], stride * dimension_offsets[dims[1]], interp_func, pb); } } for (size_t i = (begin[dims[0]] ? begin[dims[0]] + stride : 0); i <= end[dims[0]]; i += stride) { for (size_t j = (begin[dims[1]] ? begin[dims[1]] + stride : 0); j <= end[dims[1]]; j += stride) { size_t begin_offset = i * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + - begin[dims[2]] * dimension_offsets[dims[2]]; + begin[dims[2]] * dimension_offsets[dims[2]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[2]] - begin[dims[2]]) * - dimension_offsets[dims[2]], + (end[dims[2]] - begin[dims[2]]) * + dimension_offsets[dims[2]], stride * dimension_offsets[dims[2]], interp_func, pb); } } return predict_error; } - - + template typename std::enable_if::type block_interpolation(T *data, std::array begin, std::array end, const PredictorBehavior pb, @@ -392,13 +349,13 @@ namespace SZ3 { for (size_t t = (begin[dims[3]] ? begin[dims[3]] + stride2x : 0); t <= end[dims[3]]; t += stride2x) { size_t begin_offset = - begin[dims[0]] * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + + begin[dims[0]] * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + k * dimension_offsets[dims[2]] + t * dimension_offsets[dims[3]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[0]] - begin[dims[0]]) * - dimension_offsets[dims[0]], + (end[dims[0]] - begin[dims[0]]) * + dimension_offsets[dims[0]], stride * dimension_offsets[dims[0]], interp_func, pb); } } @@ -409,13 +366,13 @@ namespace SZ3 { for (size_t t = (begin[dims[3]] ? begin[dims[3]] + stride2x : 0); t <= end[dims[3]]; t += stride2x) { size_t begin_offset = - i * dimension_offsets[dims[0]] + begin[dims[1]] * dimension_offsets[dims[1]] + + i * dimension_offsets[dims[0]] + begin[dims[1]] * dimension_offsets[dims[1]] + k * dimension_offsets[dims[2]] + t * dimension_offsets[dims[3]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[1]] - begin[dims[1]]) * - dimension_offsets[dims[1]], + (end[dims[1]] - begin[dims[1]]) * + dimension_offsets[dims[1]], stride * dimension_offsets[dims[1]], interp_func, pb); } } @@ -426,57 +383,59 @@ namespace SZ3 { for (size_t t = (begin[dims[3]] ? begin[dims[3]] + stride2x : 0); t <= end[dims[3]]; t += stride2x) { size_t begin_offset = i * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + - begin[dims[2]] * dimension_offsets[dims[2]] + - t * dimension_offsets[dims[3]]; + begin[dims[2]] * dimension_offsets[dims[2]] + + t * dimension_offsets[dims[3]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[2]] - begin[dims[2]]) * - dimension_offsets[dims[2]], + (end[dims[2]] - begin[dims[2]]) * + dimension_offsets[dims[2]], stride * dimension_offsets[dims[2]], interp_func, pb); } } } - + max_error = 0; for (size_t i = (begin[dims[0]] ? begin[dims[0]] + stride : 0); i <= end[dims[0]]; i += stride) { for (size_t j = (begin[dims[1]] ? begin[dims[1]] + stride : 0); j <= end[dims[1]]; j += stride) { for (size_t k = (begin[dims[2]] ? begin[dims[2]] + stride : 0); k <= end[dims[2]]; k += stride) { size_t begin_offset = - i * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + + i * dimension_offsets[dims[0]] + j * dimension_offsets[dims[1]] + k * dimension_offsets[dims[2]] + begin[dims[3]] * dimension_offsets[dims[3]]; predict_error += block_interpolation_1d(data, begin_offset, begin_offset + - (end[dims[3]] - begin[dims[3]]) * - dimension_offsets[dims[3]], + (end[dims[3]] - begin[dims[3]]) * + dimension_offsets[dims[3]], stride * dimension_offsets[dims[3]], interp_func, pb); } } } return predict_error; } - + int interpolation_level = -1; uint blocksize; int interpolator_id; double eb_ratio = 0.5; std::vector interpolators = {"linear", "cubic"}; - std::vector quant_inds; - size_t quant_index = 0; // for decompress + int *quant_inds; + size_t quant_index = 0; double max_error; Quantizer quantizer; - Encoder encoder; - Lossless lossless; size_t num_elements; std::array global_dimensions; std::array dimension_offsets; std::vector> dimension_sequences; int direction_sequence_id; }; - - + + template + InterpolationDecomposition + make_decomposition_interpolation(const Config &conf, Quantizer quantizer) { + return InterpolationDecomposition(conf, quantizer); + } + }; - #endif diff --git a/include/SZ3/frontend/SZFastFrontend.hpp b/include/SZ3/decomposition/LorenzoRegressionDecomposition.hpp similarity index 96% rename from include/SZ3/frontend/SZFastFrontend.hpp rename to include/SZ3/decomposition/LorenzoRegressionDecomposition.hpp index 3dbca4ac..e02c823b 100644 --- a/include/SZ3/frontend/SZFastFrontend.hpp +++ b/include/SZ3/decomposition/LorenzoRegressionDecomposition.hpp @@ -1,16 +1,17 @@ -#ifndef SZ3_SZFASTFRONTEND -#define SZ3_SZFASTFRONTEND +#ifndef SZ3_LORENZO_REGRESSION_DECOMPOSITION_HPP +#define SZ3_LORENZO_REGRESSION_DECOMPOSITION_HPP /** * This module is the implementation of the prediction and quantization methods in SZ2. * It has better speed than SZFrontend since multidimensional iterator is not used. - * Currently only 3D data is supported. + * Currently only 1d and 3D data is supported. */ -#include "Frontend.hpp" +#include "Decomposition.hpp" #include "SZ3/predictor/MetaLorenzoPredictor.hpp" #include "SZ3/predictor/MetaRegressionPredictor.hpp" #include "SZ3/utils/MetaDef.hpp" +#include "SZ3/quantizer/Quantizer.hpp" #include "SZ3/encoder/HuffmanEncoder.hpp" #include "SZ3/utils/MemoryUtil.hpp" #include "SZ3/utils/Config.hpp" @@ -20,9 +21,9 @@ namespace SZ3 { using namespace SZMETA; template - class SZFastFrontend : public concepts::FrontendInterface { + class LorenzoRegressionDecomposition : public concepts::DecompositionInterface { public: - SZFastFrontend(const Config &conf, Quantizer quantizer) : + LorenzoRegressionDecomposition(const Config &conf, Quantizer quantizer) : quantizer(quantizer), params(false, conf.blockSize, conf.pred_dim, 0, conf.lorenzo, conf.lorenzo2, conf.regression, conf.absErrorBound), @@ -31,16 +32,18 @@ namespace SZ3 { if (N != 1 && N != 3) { throw std::invalid_argument("SZMeta Front only support 1D or 3D data"); } + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quatizer interface"); } - ~SZFastFrontend() { + ~LorenzoRegressionDecomposition() { clear(); } void print() {}; - std::vector compress(T *data) { + std::vector compress(const Config &conf, T *data) { if (N == 1) { return compress_1d(data); } else { @@ -48,7 +51,7 @@ namespace SZ3 { } }; - T *decompress(std::vector &quant_inds, T *dec_data) { + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) { if (N == 1) { return decompress_1d(quant_inds, dec_data); } else { @@ -153,7 +156,7 @@ namespace SZ3 { free(reg_params); reg_params = nullptr; } - quantizer.clear(); +// quantizer.clear(); } size_t size_est() { @@ -610,11 +613,12 @@ namespace SZ3 { }; - template - SZFastFrontend - make_sz_fast_frontend(const Config &conf, Predictor predictor) { - return SZFastFrontend(conf, predictor); + template + LorenzoRegressionDecomposition + make_decomposition_lorenzo_regression(const Config &conf, Quantizer quantizer) { + return LorenzoRegressionDecomposition(conf, quantizer); } + } diff --git a/include/SZ3/decomposition/NoPredictionDecomposition.hpp b/include/SZ3/decomposition/NoPredictionDecomposition.hpp new file mode 100644 index 00000000..a73d7e68 --- /dev/null +++ b/include/SZ3/decomposition/NoPredictionDecomposition.hpp @@ -0,0 +1,59 @@ +#ifndef _SZ_NO_PREDICTION_DECOMPOSITION_HPP +#define _SZ_NO_PREDICTION_DECOMPOSITION_HPP + +#include "Decomposition.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/def.hpp" + +namespace SZ3 { + template + class NoPredictionDecomposition : public concepts::DecompositionInterface { + public: + + + NoPredictionDecomposition(const Config &conf, Quantizer quantizer) : quantizer(quantizer) { + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quatizer interface"); + } + + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) { + for (size_t i = 0; i < conf.num; i++) { + dec_data[i] = quantizer.recover(0, quant_inds[i]); + } + quantizer.postdecompress_data(); + return dec_data; + } + + std::vector compress(const Config &conf, T *data) { + std::vector quant_inds(conf.num); + for (size_t i = 0; i < conf.num; i++) { + quant_inds[i] = quantizer.quantize_and_overwrite(data[i], 0); + } + quantizer.postcompress_data(); + return quant_inds; + } + + void save(uchar *&c) { + quantizer.save(c); + } + + void load(const uchar *&c, size_t &remaining_length) { + quantizer.load(c, remaining_length); + } + + private: + Quantizer quantizer; + }; + + template + NoPredictionDecomposition + make_decomposition_noprediction(const Config &conf, Quantizer quantizer) { + return NoPredictionDecomposition(conf, quantizer); + } + +}; + + +#endif + diff --git a/tools/mdz/include/TimeBasedFrontend.hpp b/include/SZ3/decomposition/TimeSeriesDecomposition.hpp similarity index 87% rename from tools/mdz/include/TimeBasedFrontend.hpp rename to include/SZ3/decomposition/TimeSeriesDecomposition.hpp index cfc8d453..7b4ea77e 100644 --- a/tools/mdz/include/TimeBasedFrontend.hpp +++ b/include/SZ3/decomposition/TimeSeriesDecomposition.hpp @@ -1,23 +1,23 @@ -#ifndef SZ_TIMEBASED_FRONTEND -#define SZ_TIMEBASED_FRONTEND - -#include -#include -#include -#include -#include -#include -#include -#include +#ifndef SZ_TIME_SERIES_DECOMPOSITION_HPP +#define SZ_TIME_SERIES_DECOMPOSITION_HPP + +#include "Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/Iterator.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/MemoryUtil.hpp" namespace SZ3 { template - class TimeBasedFrontend : public concepts::FrontendInterface { + class TimeSeriesDecomposition : public concepts::DecompositionInterface { public: - TimeBasedFrontend(const Config &conf, Predictor predictor, Quantizer quantizer, T *data_ts0) : + TimeSeriesDecomposition(const Config &conf, Predictor predictor, Quantizer quantizer, T *data_ts0) : fallback_predictor(LorenzoPredictor(conf.absErrorBound)), predictor(predictor), quantizer(quantizer), @@ -31,9 +31,9 @@ namespace SZ3 { global_dimensions[1] = conf.dims[1]; } - ~TimeBasedFrontend() = default; + ~TimeSeriesDecomposition() = default; - std::vector compress(T *data) { + std::vector compress(const Config &conf, T *data) { std::vector quant_inds(num_elements); size_t quant_count = 0; if (data_ts0 != nullptr) { @@ -99,7 +99,7 @@ namespace SZ3 { return quant_inds; } - T *decompress(std::vector &quant_inds, T *dec_data) { + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) { int const *quant_inds_pos = (int const *) quant_inds.data(); std::array intra_block_dims; @@ -184,22 +184,8 @@ namespace SZ3 { quantizer.load(c, remaining_length); } - void print() { -// predictor.print(); -// quantizer.print(); - } - - void clear() { - predictor.clear(); - fallback_predictor.clear(); - quantizer.clear(); - } - int get_radius() const { return quantizer.get_radius(); } - size_t get_num_elements() const { return num_elements; }; - - size_t size_est() { return 0; }; private: Predictor predictor; LorenzoPredictor fallback_predictor; @@ -212,9 +198,9 @@ namespace SZ3 { }; template - TimeBasedFrontend - make_sz3_timebased_frontend(const Config &conf, Predictor predictor, Quantizer quantizer, T *data_ts0) { - return TimeBasedFrontend(conf, predictor, quantizer, data_ts0); + TimeSeriesDecomposition + make_decomposition_timeseries(const Config &conf, Predictor predictor, Quantizer quantizer, T *data_ts0) { + return TimeSeriesDecomposition(conf, predictor, quantizer, data_ts0); } } diff --git a/include/SZ3/frontend/SZGeneralFrontend.hpp b/include/SZ3/frontend/SZGeneralFrontend.hpp deleted file mode 100644 index 2d817cb3..00000000 --- a/include/SZ3/frontend/SZGeneralFrontend.hpp +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef SZ3_FRONTEND -#define SZ3_FRONTEND -/** - * This module is the implementation of general frontend in SZ3 - */ - -#include "Frontend.hpp" -#include "SZ3/def.hpp" -#include "SZ3/predictor/Predictor.hpp" -#include "SZ3/predictor/LorenzoPredictor.hpp" -#include "SZ3/quantizer/Quantizer.hpp" -#include "SZ3/utils/Iterator.hpp" -#include "SZ3/utils/Config.hpp" -#include "SZ3/utils/MemoryUtil.hpp" - -namespace SZ3 { - - - template - class SZGeneralFrontend : public concepts::FrontendInterface { - public: - - SZGeneralFrontend(const Config &conf, Predictor predictor, Quantizer quantizer) : - fallback_predictor(LorenzoPredictor(conf.absErrorBound)), - predictor(predictor), - quantizer(quantizer), - block_size(conf.blockSize), - num_elements(conf.num) { - std::copy_n(conf.dims.begin(), N, global_dimensions.begin()); - } - - ~SZGeneralFrontend() = default; - - std::vector compress(T *data) { - std::vector quant_inds(num_elements); - auto block_range = std::make_shared>( - data, std::begin(global_dimensions), std::end(global_dimensions), block_size, 0); - - auto element_range = std::make_shared>( - data, std::begin(global_dimensions), std::end(global_dimensions), 1, 0); - - predictor.precompress_data(block_range->begin()); - quantizer.precompress_data(); - size_t quant_count = 0; - for (auto block = block_range->begin(); block != block_range->end(); ++block) { - - element_range->update_block_range(block, block_size); - - concepts::PredictorInterface *predictor_withfallback = &predictor; - if (!predictor.precompress_block(element_range)) { - predictor_withfallback = &fallback_predictor; - } - predictor_withfallback->precompress_block_commit(); - - for (auto element = element_range->begin(); element != element_range->end(); ++element) { - quant_inds[quant_count++] = quantizer.quantize_and_overwrite( - *element, predictor_withfallback->predict(element)); - } - } - - predictor.postcompress_data(block_range->begin()); - quantizer.postcompress_data(); - return quant_inds; - } - - T *decompress(std::vector &quant_inds, T *dec_data) { - - int const *quant_inds_pos = (int const *) quant_inds.data(); - std::array intra_block_dims; -// auto dec_data = new T[num_elements]; - auto block_range = std::make_shared>( - dec_data, std::begin(global_dimensions), std::end(global_dimensions), block_size, 0); - - auto element_range = std::make_shared>( - dec_data, std::begin(global_dimensions), std::end(global_dimensions), 1, 0); - - predictor.predecompress_data(block_range->begin()); - quantizer.predecompress_data(); - - for (auto block = block_range->begin(); block != block_range->end(); ++block) { - - element_range->update_block_range(block, block_size); - - concepts::PredictorInterface *predictor_withfallback = &predictor; - if (!predictor.predecompress_block(element_range)) { - predictor_withfallback = &fallback_predictor; - } - for (auto element = element_range->begin(); element != element_range->end(); ++element) { - *element = quantizer.recover(predictor_withfallback->predict(element), *(quant_inds_pos++)); - } - } - predictor.postdecompress_data(block_range->begin()); - quantizer.postdecompress_data(); - return dec_data; - } - - void save(uchar *&c) { - write(global_dimensions.data(), N, c); - write(block_size, c); - - predictor.save(c); - quantizer.save(c); - } - - void load(const uchar *&c, size_t &remaining_length) { - read(global_dimensions.data(), N, c, remaining_length); - num_elements = 1; - for (const auto &d: global_dimensions) { - num_elements *= d; - } - read(block_size, c, remaining_length); - predictor.load(c, remaining_length); - quantizer.load(c, remaining_length); - } - - size_t size_est() { - return quantizer.size_est(); - } - - void print() { -// predictor.print(); -// quantizer.print(); - } - - void clear() { - predictor.clear(); - fallback_predictor.clear(); - quantizer.clear(); - } - - int get_radius() { return quantizer.get_radius(); } - - private: - Predictor predictor; - LorenzoPredictor fallback_predictor; - Quantizer quantizer; - uint block_size; - size_t num_elements; - std::array global_dimensions; - - }; - - template - SZGeneralFrontend - make_sz_general_frontend(const Config &conf, Predictor predictor, Quantizer quantizer) { - return SZGeneralFrontend(conf, predictor, quantizer); - } -} - -#endif diff --git a/include/SZ3/lossless/Lossless.hpp b/include/SZ3/lossless/Lossless.hpp index e3f9f52d..227583d2 100644 --- a/include/SZ3/lossless/Lossless.hpp +++ b/include/SZ3/lossless/Lossless.hpp @@ -14,26 +14,26 @@ namespace SZ3::concepts { */ class LosslessInterface { public: - virtual void postcompress_data(uchar *data) = 0; - - virtual void postdecompress_data(uchar *data) = 0; /** * compress data with lossless compressors - * @param data data to be compressed - * @param dataLength length (in bytes) of the data to be compressed - * @param outSize compressed size (in bytes) - * @return compressed data + * @param src data to be compressed + * @param srcLen length (in bytes) of the data to be compressed + * @param dst compressed data + * @param dstCap capacity (in bytes) for storing the compressed data + * @return length (in bytes) of the data compressed */ - virtual uchar *compress(uchar *data, size_t dataLength, size_t &outSize) = 0; + virtual size_t compress(uchar *src, size_t srcLen, uchar *dst, size_t dstCap) = 0; /** * reverse of compress(), decompress the data with lossless compressors - * @param data data to be decompressed - * @param compressedSize length (in bytes) of the data to be decompressed (as input) or the data decompressed (as output). - * @return decompressed data + * @param src data to be decompressed + * @param srcLen length (in bytes) of the data to be decompressed (as input) or the data decompressed (as output). + * @param dst decompressed data + * @param dstCap capacity (in bytes) for storing the decompressed data (in bytes) + * @return length (in bytes) of the data decompressed */ - virtual uchar *decompress(const uchar *data, size_t &compressedSize) = 0; + virtual size_t decompress(const uchar *src, const size_t srcLen, uchar *dst, size_t dstCap) = 0; }; } diff --git a/include/SZ3/lossless/Lossless_bypass.hpp b/include/SZ3/lossless/Lossless_bypass.hpp index 9b7b75b1..2f525cc2 100644 --- a/include/SZ3/lossless/Lossless_bypass.hpp +++ b/include/SZ3/lossless/Lossless_bypass.hpp @@ -13,20 +13,17 @@ namespace SZ3 { class Lossless_bypass : public concepts::LosslessInterface { - - public: - - void postcompress_data(uchar *data) {}; - - void postdecompress_data(uchar *data) {}; - - uchar *compress(uchar *data, size_t dataLength, size_t &outSize) { - outSize = dataLength; - return data; + + public: + + size_t compress(uchar *src, size_t srcLen, uchar *dst, size_t dstCap) { + dst = src; + return srcLen; } - - uchar *decompress(const uchar *data, size_t &compressedSize) { - return (uchar *) data; + + size_t decompress(const uchar *src, const size_t srcLen, uchar *dst, size_t dstCap) { + dst = (uchar *) src; + return srcLen; } }; } diff --git a/include/SZ3/lossless/Lossless_zstd.hpp b/include/SZ3/lossless/Lossless_zstd.hpp index 6cb36a9f..482035c1 100644 --- a/include/SZ3/lossless/Lossless_zstd.hpp +++ b/include/SZ3/lossless/Lossless_zstd.hpp @@ -13,44 +13,38 @@ namespace SZ3 { class Lossless_zstd : public concepts::LosslessInterface { - - public: + + public: Lossless_zstd() = default; - + Lossless_zstd(int comp_level) : compression_level(comp_level) {}; - - uchar *compress(uchar *data, size_t dataLength, size_t &outSize) { - size_t estimatedCompressedSize = std::max(size_t(dataLength * 1.2), size_t(400)); - uchar *compressBytes = new uchar[estimatedCompressedSize]; - uchar *compressBytesPos = compressBytes; - write(dataLength, compressBytesPos); - - outSize = ZSTD_compress(compressBytesPos, estimatedCompressedSize, data, dataLength, - compression_level); - outSize += sizeof(size_t); - return compressBytes; + + size_t compress(uchar *src, size_t srcLen, uchar *dst, size_t dstCap) { +// size_t estimatedCompressedSize = std::max(size_t(srcLen * 1.2), size_t(400)); +// uchar *compressBytes = new uchar[estimatedCompressedSize]; +// uchar *dstPos = dst; +// write(srcLen, dstPos); +// if (dstCap < srcLen) { +// throw std::invalid_argument( +// "dstCap not large enough for zstd"); +// } + return ZSTD_compress(dst, dstCap, src, srcLen, compression_level); +// dstLen += sizeof(size_t); +// return compressBytes; } - - void postcompress_data(uchar *data) { - delete[] data; + + size_t decompress(const uchar *src, const size_t srcLen, uchar *dst, size_t dstCap) { +// const uchar *dataPos = data; +// size_t dataLength = 0; +// read(dataLength, dataPos, compressedSize); + +// uchar *oriData = new uchar[dataLength]; + return ZSTD_decompress(dst, dstCap, src, srcLen); +// compressedSize = dataLength; +// return oriData; } - - uchar *decompress(const uchar *data, size_t &compressedSize) { - const uchar *dataPos = data; - size_t dataLength = 0; - read(dataLength, dataPos, compressedSize); - - uchar *oriData = new uchar[dataLength]; - ZSTD_decompress(oriData, dataLength, dataPos, compressedSize); - compressedSize = dataLength; - return oriData; - } - - void postdecompress_data(uchar *data) { - delete[] data; - } - - private: + + private: int compression_level = 3; //default setting of level is 3 }; } diff --git a/include/SZ3/predictor/ComposedPredictor.hpp b/include/SZ3/predictor/ComposedPredictor.hpp index 2f380112..81d68232 100644 --- a/include/SZ3/predictor/ComposedPredictor.hpp +++ b/include/SZ3/predictor/ComposedPredictor.hpp @@ -167,12 +167,12 @@ namespace SZ3 { predict_error.resize(predictors.size()); } - void clear() { - for (auto &pred:predictors) { - pred->clear(); - } - selection.clear(); - } +// void clear() { +// for (auto &pred:predictors) { +// pred->clear(); +// } +// selection.clear(); +// } private: std::vector>> diff --git a/include/SZ3/predictor/LorenzoPredictor.hpp b/include/SZ3/predictor/LorenzoPredictor.hpp index 98bbc837..1b710d9c 100644 --- a/include/SZ3/predictor/LorenzoPredictor.hpp +++ b/include/SZ3/predictor/LorenzoPredictor.hpp @@ -94,7 +94,7 @@ namespace SZ3 { return do_predict(iter); } - void clear() {} +// void clear() {} protected: T noise = 0; diff --git a/include/SZ3/predictor/PolyRegressionPredictor.hpp b/include/SZ3/predictor/PolyRegressionPredictor.hpp index 4e2b40aa..4a496131 100644 --- a/include/SZ3/predictor/PolyRegressionPredictor.hpp +++ b/include/SZ3/predictor/PolyRegressionPredictor.hpp @@ -175,15 +175,15 @@ namespace SZ3 { std::cout << "2-Layer Regression predictor, poly term eb = " << quantizer_poly.get_eb() << "\n"; } - void clear() { - quantizer_independent.clear(); - quantizer_liner.clear(); - quantizer_poly.clear(); - regression_coeff_quant_inds.clear(); - regression_coeff_index = 0; - current_coeffs = {0}; - prev_coeffs = {0}; - } +// void clear() { +// quantizer_independent.clear(); +// quantizer_liner.clear(); +// quantizer_poly.clear(); +// regression_coeff_quant_inds.clear(); +// regression_coeff_index = 0; +// current_coeffs = {0}; +// prev_coeffs = {0}; +// } private: LinearQuantizer quantizer_independent, quantizer_liner, quantizer_poly; diff --git a/include/SZ3/predictor/Predictor.hpp b/include/SZ3/predictor/Predictor.hpp index 1da060e9..709a5ef6 100644 --- a/include/SZ3/predictor/Predictor.hpp +++ b/include/SZ3/predictor/Predictor.hpp @@ -8,10 +8,13 @@ namespace SZ3::concepts { /** - * Data prediction - * example: lorenzo, regression, interpolation - * This will be changed to DecompositionTypeI - * See Frontend.hpp for details + * data prediction interface + * + * difference between Prediction and Decomposition interfaces: + * Prediction: + * prediction function takes scalar value (e.g, single data point) + * Decomposition: + * prediction function takes multidimensional tensors (e.g, whole input) * @tparam T original data type * @tparam N original data dimension */ @@ -76,7 +79,7 @@ namespace SZ3::concepts { virtual void print() const = 0; - virtual void clear() = 0; +// virtual void clear() = 0; }; /** diff --git a/include/SZ3/predictor/RegressionPredictor.hpp b/include/SZ3/predictor/RegressionPredictor.hpp index 1293b75b..18cc4d7c 100644 --- a/include/SZ3/predictor/RegressionPredictor.hpp +++ b/include/SZ3/predictor/RegressionPredictor.hpp @@ -165,14 +165,14 @@ namespace SZ3 { std::cout << std::endl; } - void clear() { - quantizer_liner.clear(); - quantizer_independent.clear(); - regression_coeff_quant_inds.clear(); - regression_coeff_index = 0; - current_coeffs = {0}; - prev_coeffs = {0}; - } +// void clear() { +// quantizer_liner.clear(); +// quantizer_independent.clear(); +// regression_coeff_quant_inds.clear(); +// regression_coeff_index = 0; +// current_coeffs = {0}; +// prev_coeffs = {0}; +// } std::array get_current_coeffs() { return current_coeffs; diff --git a/include/SZ3/quantizer/IntegerQuantizer.hpp b/include/SZ3/quantizer/IntegerQuantizer.hpp index e279c106..caf6e844 100644 --- a/include/SZ3/quantizer/IntegerQuantizer.hpp +++ b/include/SZ3/quantizer/IntegerQuantizer.hpp @@ -30,32 +30,6 @@ namespace SZ3 { error_bound_reciprocal = 1.0 / eb; } - // quantize the data with a prediction value, and returns the quantization index - int quantize(T data, T pred) { - T diff = data - pred; - auto quant_index = (int64_t) (fabs(diff) * this->error_bound_reciprocal) + 1; - if (quant_index < this->radius * 2) { - quant_index >>= 1; - int half_index = quant_index; - quant_index <<= 1; - int quant_index_shifted; - if (diff < 0) { - quant_index = -quant_index; - quant_index_shifted = this->radius - half_index; - } else { - quant_index_shifted = this->radius + half_index; - } - T decompressed_data = pred + quant_index * this->error_bound; - if (fabs(decompressed_data - data) > this->error_bound) { - return 0; - } else { - return quant_index_shifted; - } - } else { - return 0; - } - } - // quantize the data with a prediction value, and returns the quantization index and the decompressed data // int quantize(T data, T pred, T& dec_data); int quantize_and_overwrite(T &data, T pred) { @@ -86,6 +60,13 @@ namespace SZ3 { } } + /** + * For metaLorenzo only, will be removed together with metalorenzo + * @param ori + * @param pred + * @param dest + * @return + */ int quantize_and_overwrite(T ori, T pred, T &dest) { T diff = ori - pred; auto quant_index = (int64_t) (fabs(diff) * this->error_bound_reciprocal) + 1; @@ -174,10 +155,10 @@ namespace SZ3 { printf("[IntegerQuantizer] error_bound = %.8G, radius = %d, unpred = %lu\n", error_bound, radius, unpred.size()); } - void clear() { - unpred.clear(); - index = 0; - } +// void clear() { +// unpred.clear(); +// index = 0; +// } virtual void postcompress_data() { diff --git a/include/SZ3/quantizer/Quantizer.hpp b/include/SZ3/quantizer/Quantizer.hpp index 5bffd681..99641da1 100644 --- a/include/SZ3/quantizer/Quantizer.hpp +++ b/include/SZ3/quantizer/Quantizer.hpp @@ -15,14 +15,6 @@ namespace SZ3::concepts { virtual ~QuantizerInterface() = default; - /** - * quantize the error (error=data-pred) based on error bound - * @param data single data point - * @param pred predicted value for the data point - * @return quantized error - */ - virtual int quantize(T data, T pred) = 0; - /** * quantize the error (error=data-pred) based on error bound, and overwrite the data with reconstructed value * @param data single data point @@ -39,10 +31,6 @@ namespace SZ3::concepts { */ virtual T recover(T pred, int quant_index) = 0; - /** - * reset quantizer to initial state - */ - virtual void clear() = 0; /** ** serialize the quantizer and store it to a buffer diff --git a/include/SZ3/utils/ByteUtil.hpp b/include/SZ3/utils/ByteUtil.hpp index f408fd67..7f3824f7 100644 --- a/include/SZ3/utils/ByteUtil.hpp +++ b/include/SZ3/utils/ByteUtil.hpp @@ -8,6 +8,7 @@ #include "SZ3/def.hpp" #include #include +#include namespace SZ3 { diff --git a/include/SZ3/utils/Config.hpp b/include/SZ3/utils/Config.hpp index 324d9b2b..20e03044 100644 --- a/include/SZ3/utils/Config.hpp +++ b/include/SZ3/utils/Config.hpp @@ -9,30 +9,42 @@ #include #include #include +#include "SZ3/version.hpp" #include "SZ3/def.hpp" -#include "MemoryUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" #include "SZ3/utils/inih/INIReader.h" -namespace SZ3 { +#define SZ_FLOAT 0 +#define SZ_DOUBLE 1 +#define SZ_UINT8 2 +#define SZ_INT8 3 +#define SZ_UINT16 4 +#define SZ_INT16 5 +#define SZ_UINT32 6 +#define SZ_INT32 7 +#define SZ_UINT64 8 +#define SZ_INT64 9 +namespace SZ3 { + enum EB { EB_ABS, EB_REL, EB_PSNR, EB_L2NORM, EB_ABS_AND_REL, EB_ABS_OR_REL }; constexpr const char *EB_STR[] = {"ABS", "REL", "PSNR", "NORM", "ABS_AND_REL", "ABS_OR_REL"}; constexpr EB EB_OPTIONS[] = {EB_ABS, EB_REL, EB_PSNR, EB_L2NORM, EB_ABS_AND_REL, EB_ABS_OR_REL}; - + enum ALGO { - ALGO_LORENZO_REG, ALGO_INTERP_LORENZO, ALGO_INTERP + ALGO_LORENZO_REG, ALGO_INTERP_LORENZO, ALGO_INTERP, ALGO_NOPRED, }; - constexpr const char *ALGO_STR[] = {"ALGO_LORENZO_REG", "ALGO_INTERP_LORENZO", "ALGO_INTERP"}; - constexpr const ALGO ALGO_OPTIONS[] = {ALGO_LORENZO_REG, ALGO_INTERP_LORENZO, ALGO_INTERP}; - + constexpr const char *ALGO_STR[] = {"ALGO_LORENZO_REG", "ALGO_INTERP_LORENZO", "ALGO_INTERP", "ALGO_NOPRED"}; + constexpr const ALGO ALGO_OPTIONS[] = {ALGO_LORENZO_REG, ALGO_INTERP_LORENZO, ALGO_INTERP, ALGO_NOPRED}; + enum INTERP_ALGO { INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC }; constexpr const char *INTERP_ALGO_STR[] = {"INTERP_ALGO_LINEAR", "INTERP_ALGO_CUBIC"}; constexpr INTERP_ALGO INTERP_ALGO_OPTIONS[] = {INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC}; - + template const char *enum2Str(T e) { if (std::is_same::value) { @@ -46,36 +58,43 @@ namespace SZ3 { exit(0); } } - + class Config { - public: + public: template - Config(Dims ... args) { + Config(Dims... args) { dims = std::vector{static_cast(std::forward(args))...}; - N = dims.size(); - num = std::accumulate(dims.begin(), dims.end(), (size_t) 1, std::multiplies()); - blockSize = (N == 1 ? 128 : (N == 2 ? 16 : 6)); - pred_dim = N; - stride = blockSize; + setDims(dims.begin(), dims.end()); } - + template size_t setDims(Iter begin, Iter end) { - dims = std::vector(begin, end); + auto dims_ = std::vector(begin, end); + dims.clear(); + for (auto dim : dims_) { + if (dim > 1) { + dims.push_back(dim); + } + } + if (dims.empty()) { + dims = {1}; + } N = dims.size(); num = std::accumulate(dims.begin(), dims.end(), (size_t) 1, std::multiplies()); pred_dim = N; + blockSize = (N == 1 ? 128 : (N == 2 ? 16 : 6)); + stride = blockSize; return num; } - + void loadcfg(const std::string &cfgpath) { INIReader cfg(cfgpath); - + if (cfg.ParseError() != 0) { std::cout << "Can't load cfg file " << cfgpath << std::endl; exit(0); } - + auto cmprAlgoStr = cfg.Get("GlobalSettings", "CmprAlgo", ""); if (cmprAlgoStr == ALGO_STR[ALGO_LORENZO_REG]) { cmprAlgo = ALGO_LORENZO_REG; @@ -83,6 +102,8 @@ namespace SZ3 { cmprAlgo = ALGO_INTERP_LORENZO; } else if (cmprAlgoStr == ALGO_STR[ALGO_INTERP]) { cmprAlgo = ALGO_INTERP; + } else if (cmprAlgoStr == ALGO_STR[ALGO_NOPRED]) { + cmprAlgo = ALGO_NOPRED; } auto ebModeStr = cfg.Get("GlobalSettings", "ErrorBoundMode", ""); if (ebModeStr == EB_STR[EB_ABS]) { @@ -102,13 +123,13 @@ namespace SZ3 { relErrorBound = cfg.GetReal("GlobalSettings", "RelErrorBound", relErrorBound); psnrErrorBound = cfg.GetReal("GlobalSettings", "PSNRErrorBound", psnrErrorBound); l2normErrorBound = cfg.GetReal("GlobalSettings", "L2NormErrorBound", l2normErrorBound); - + openmp = cfg.GetBoolean("GlobalSettings", "OpenMP", openmp); lorenzo = cfg.GetBoolean("AlgoSettings", "Lorenzo", lorenzo); lorenzo2 = cfg.GetBoolean("AlgoSettings", "Lorenzo2ndOrder", lorenzo2); regression = cfg.GetBoolean("AlgoSettings", "Regression", regression); regression2 = cfg.GetBoolean("AlgoSettings", "Regression2ndOrder", regression2); - + auto interpAlgoStr = cfg.Get("AlgoSettings", "InterpolationAlgo", ""); if (interpAlgoStr == INTERP_ALGO_STR[INTERP_ALGO_LINEAR]) { interpAlgo = INTERP_ALGO_LINEAR; @@ -119,12 +140,13 @@ namespace SZ3 { interpBlockSize = cfg.GetInteger("AlgoSettings", "InterpolationBlockSize", interpBlockSize); blockSize = cfg.GetInteger("AlgoSettings", "BlockSize", blockSize); quantbinCnt = cfg.GetInteger("AlgoSettings", "QuantizationBinTotal", quantbinCnt); - - + } - - + void save(unsigned char *&c) { + auto c0 = c; + write(sz3MagicNumber, c); + write(sz3DataVer, c); write(N, c); write(dims.data(), dims.size(), c); write(num, c); @@ -146,9 +168,19 @@ namespace SZ3 { write(stride, c); write(pred_dim, c); write(openmp, c); + write(dataType, c); + if (c - c0 > size_est()) { + throw std::invalid_argument("Config saved size is larger than estimated size"); + } else { + //for padding + c = c0 + size_est(); + } }; - + void load(const unsigned char *&c) { + auto c0 = c; + read(sz3MagicNumber, c); + read(sz3DataVer, c); read(N, c); dims.resize(N); read(dims.data(), N, c); @@ -171,22 +203,61 @@ namespace SZ3 { read(stride, c); read(pred_dim, c); read(openmp, c); + read(dataType, c); + if (c - c0 > size_est()) { + throw std::invalid_argument("Config loaded size is larger than estimated size"); + } else { + //for padding + c = c0 + size_est(); + } } - + void print() { + printf("===================== Begin SZ3 Configuration =====================\n"); + printf("sz3MagicNumber = %u\n", sz3MagicNumber); + printf("sz3DataVer = %s\n", versionStr(sz3DataVer).data()); + printf("N = %d\n", N); + printf("dims = "); + for (auto dim : dims) { + printf("%zu ", dim); + } + printf("\nnum = %zu\n", num); printf("CmprAlgo = %s\n", enum2Str((ALGO) cmprAlgo)); + printf("ErrorBoundMode = %s\n", enum2Str((EB) errorBoundMode)); + printf("AbsErrorBound = %f\n", absErrorBound); + printf("RelErrorBound = %f\n", relErrorBound); + printf("PSNRErrorBound = %f\n", psnrErrorBound); + printf("L2NormErrorBound = %f\n", l2normErrorBound); + printf("Lorenzo = %d\n", lorenzo); + printf("Lorenzo2ndOrder = %d\n", lorenzo2); + printf("Regression = %d\n", regression); + printf("Regression2ndOrder = %d\n", regression2); + printf("OpenMP = %d\n", openmp); + printf("DataType = %d\n", dataType); + printf("Lossless = %d\n", lossless); + printf("Encoder = %d\n", encoder); + printf("InterpolationAlgo = %s\n", enum2Str((INTERP_ALGO) interpAlgo)); + printf("InterpolationDirection = %d\n", interpDirection); + printf("InterpolationBlockSize = %d\n", interpBlockSize); + printf("QuantizationBinTotal = %d\n", quantbinCnt); + printf("BlockSize = %d\n", blockSize); + printf("Stride = %d\n", stride); + printf("PredDim = %d\n", pred_dim); + printf("===================== End SZ3 Configuration =====================\n"); } - + static size_t size_est() { - return sizeof(size_t) * 5 + sizeof(double) * 4 + sizeof(bool) * 5 + sizeof(uint8_t) * 6 + sizeof(int) * 5 + 50; //50 is for redundancy + return sizeof(Config) + sizeof(size_t) * 5 + 32; //sizeof(size_t) * 5 is for dims vector, 32 is for redundancy } - + + uint32_t sz3MagicNumber = SZ3_MAGIC_NUMBER; + uint32_t sz3DataVer = versionInt(SZ3_DATA_VER); char N = 0; std::vector dims; size_t num = 0; uint8_t cmprAlgo = ALGO_INTERP_LORENZO; uint8_t errorBoundMode = EB_ABS; - double absErrorBound = 0.0; + double absErrorBound = 1e-3; double relErrorBound = 0.0; double psnrErrorBound = 0.0; double l2normErrorBound = 0.0; @@ -195,6 +266,7 @@ namespace SZ3 { bool regression = true; bool regression2 = false; bool openmp = false; + uint8_t dataType = SZ_FLOAT; // dataType is only used in HDF5 filter uint8_t lossless = 1; // 0-> skip lossless(use lossless_bypass); 1-> zstd uint8_t encoder = 1;// 0-> skip encoder; 1->HuffmanEncoder; 2->ArithmeticEncoder uint8_t interpAlgo = INTERP_ALGO_CUBIC; @@ -204,10 +276,9 @@ namespace SZ3 { int blockSize = 0; int stride = 0;//not used now int pred_dim = 0; // not used now - + }; - - + } #endif //SZ_CONFIG_HPP diff --git a/include/SZ3/utils/Extraction.hpp b/include/SZ3/utils/Extraction.hpp index 0fba6258..a3a42ee7 100644 --- a/include/SZ3/utils/Extraction.hpp +++ b/include/SZ3/utils/Extraction.hpp @@ -5,7 +5,6 @@ #ifndef SZ_EXTRACTION_HPP #define SZ_EXTRACTION_HPP - namespace SZ3 { template diff --git a/tools/mdz/include/KmeansUtil.hpp b/include/SZ3/utils/KmeansUtil.hpp similarity index 99% rename from tools/mdz/include/KmeansUtil.hpp rename to include/SZ3/utils/KmeansUtil.hpp index 88cdf00a..8658a74f 100644 --- a/tools/mdz/include/KmeansUtil.hpp +++ b/include/SZ3/utils/KmeansUtil.hpp @@ -12,8 +12,8 @@ #include #include #include -#include -#include +#include "Timer.hpp" +#include "SZ3/def.hpp" namespace SZ3 { diff --git a/include/SZ3/utils/MemoryUtil.hpp b/include/SZ3/utils/MemoryUtil.hpp index 9a29963a..44233a76 100644 --- a/include/SZ3/utils/MemoryUtil.hpp +++ b/include/SZ3/utils/MemoryUtil.hpp @@ -4,9 +4,12 @@ #ifndef SZ_MEMORYOPS_HPP #define SZ_MEMORYOPS_HPP + #include #include +#include #include "SZ3/def.hpp" + namespace SZ3 { // read array template @@ -54,5 +57,15 @@ namespace SZ3 { memcpy(compressed_data_pos, &var, sizeof(T1)); compressed_data_pos += sizeof(T1); } + + void write_str(std::string const &var, uchar *&compressed_data_pos) { + strcpy(reinterpret_cast(compressed_data_pos), var.c_str()); + compressed_data_pos += var.size() + 1; + } + + void read_str(std::string &var, uchar const *&compressed_data_pos) { + var = std::string(reinterpret_cast(compressed_data_pos)); + compressed_data_pos += var.size() + 1; + } } #endif //SZ_MEMORYOPS_HPP diff --git a/include/SZ3/version.hpp.in b/include/SZ3/version.hpp.in index 07ef1273..e944c79d 100644 --- a/include/SZ3/version.hpp.in +++ b/include/SZ3/version.hpp.in @@ -1,10 +1,13 @@ // // DO NOT MODIFY version.hpp, it is automatically generated by cmake -// The version is controlled in CMakeLists.txt +// Change the version numbers in CMakeLists.txt // #ifndef SZ3_VERSION_HPP #define SZ3_VERSION_HPP +#include + +#define SZ3_MAGIC_NUMBER 0xF342F310 #define SZ3_NAME "@PROJECT_NAME@" #define SZ3_VER "@PROJECT_VERSION@" @@ -13,4 +16,21 @@ #define SZ3_VER_PATCH @PROJECT_VERSION_PATCH@ #define SZ3_VER_TWEAK @PROJECT_VERSION_TWEAK@ +#define SZ3_DATA_VER "@SZ3_DATA_VERSION@" + +uint32_t versionInt(const std::string& version) { + uint32_t major = 0, minor = 0, patch = 0; + char dot; + std::stringstream ss(version); + ss >> major >> dot >> minor >> dot >> patch; + return (major << 24) | (minor << 16) | (patch<<8); +} + +std::string versionStr(uint32_t version) { + uint32_t major = (version >> 24) & 0xFF; + uint32_t minor = (version >> 16) & 0xFF; + uint32_t patch = (version >> 8) & 0xFF; + return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch); +} + #endif //SZ3_VERSION_HPP diff --git a/tools/H5Z-SZ3/include/H5Z_SZ3.hpp b/tools/H5Z-SZ3/include/H5Z_SZ3.hpp index cd26fe87..16582349 100644 --- a/tools/H5Z-SZ3/include/H5Z_SZ3.hpp +++ b/tools/H5Z-SZ3/include/H5Z_SZ3.hpp @@ -1,36 +1,15 @@ -// -// Created by arham23 on 2/8/22. -// - #ifndef SZ3_H5Z_SZ3_H #define SZ3_H5Z_SZ3_H #define H5Z_FILTER_SZ3 32024 -#define SZ_FLOAT 0 -#define SZ_DOUBLE 1 -#define SZ_UINT8 2 -#define SZ_INT8 3 -#define SZ_UINT16 4 -#define SZ_INT16 5 -#define SZ_UINT32 6 -#define SZ_INT32 7 -#define SZ_UINT64 8 -#define SZ_INT64 9 #include "hdf5.h" -#include -#include -#include -#include +#include #include - - - - -#define LITTLE_ENDIAN_SYSTEM 0 -#define BIG_ENDIAN_SYSTEM 1 -#define LITTLE_ENDIAN_DATA 0 -#define BIG_ENDIAN_DATA 1 +#include +#include +#include +#include "SZ3/api/sz.hpp" #ifdef __cplusplus extern "C" { @@ -48,48 +27,23 @@ do { \ #define H5Z_SZ_PUSH_AND_GOTO(MAJ, MIN, RET, MSG) \ do \ { \ - H5Epush(H5E_DEFAULT,__FILE__,_funcname_,__LINE__,H5Z_SZ_ERRCLASS,MAJ,MIN,MSG); \ - return RET; \ + H5Epush(H5E_DEFAULT,__FILE__,_funcname_,__LINE__,H5Z_SZ_ERRCLASS,MAJ,MIN,MSG); \ + return RET; \ } while(0) -extern int sysEndianType; -extern int dataEndianType; - -void SZ_refreshDimForCdArray(int dataType, size_t old_cd_nelmts, unsigned int *old_cd_values, size_t* new_cd_nelmts, unsigned int **new_cd_values, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -void SZ_errConfigToCdArray(size_t* cd_nelmts, unsigned int **cd_values, int error_bound_mode, double abs_error, double rel_error, double l2normErrorBound, double psnr); static herr_t H5Z_sz3_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id); -static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t* buf_size, void** buf); +static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf); -void SZ_cdArrayToMetaData(size_t cd_nelmts, const unsigned int cd_values[], int* dimSize, int* dataType, size_t* r5, size_t* r4, size_t* r3, size_t* r2, size_t* r1); +const void *H5PLget_plugin_info(void); -void SZ_cdArrayToMetaDataErr(size_t cd_nelmts, const unsigned int cd_values[], int* dimSize, int* dataType, size_t* r5, size_t* r4, size_t* r3, size_t* r2, size_t* r1, - int* error_bound_mode, double* abs_error, double* rel_error, double* l2norm_error, double* psnr); +herr_t set_SZ3_conf_to_H5(const hid_t propertyList, SZ3::Config &conf); -void SZ_copymetaDataToCdArray(size_t* cd_nelmts, unsigned int *cd_values, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -int checkCDValuesWithErrors(size_t cd_nelmts, const unsigned int cd_values[]); - -size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); -int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); -void init_dims_chunk(int dim, hsize_t dims[5], hsize_t chunk[5], size_t nbEle, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); - -double bytesToDouble(unsigned char* bytes); -void doubleToBytes(unsigned char *b, double num); - -void longToBytes_bigEndian(unsigned char *b, uint64_t num) ; - -int bytesToInt_bigEndian(unsigned char* bytes); -int64_t bytesToLong_bigEndian(unsigned char* b); - -void detectSysEndianType(); -void symTransform_8bytes(unsigned char data[8]); - -int filterDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t* correctedDimension); +herr_t get_SZ3_conf_from_H5(const hid_t propertyList, SZ3::Config &conf); #ifdef __cplusplus } #endif + #endif //SZ3_H5Z_SZ3_H diff --git a/tools/H5Z-SZ3/src/H5Z_SZ3.cpp b/tools/H5Z-SZ3/src/H5Z_SZ3.cpp index adee0cc6..e7487d1e 100644 --- a/tools/H5Z-SZ3/src/H5Z_SZ3.cpp +++ b/tools/H5Z-SZ3/src/H5Z_SZ3.cpp @@ -1,41 +1,23 @@ -// -// Created by arham23 on 2/8/22. -// - #include -#include "H5Z_SZ3.hpp" #include +#include #include "H5PLextern.h" -#include "SZ3/api/sz.hpp" -#include "SZ3/utils/ByteUtil.hpp" - - +#include "H5Z_SZ3.hpp" -int sysEndianType = LITTLE_ENDIAN_SYSTEM; -int dataEndianType = LITTLE_ENDIAN_DATA; hid_t H5Z_SZ_ERRCLASS = -1; -using namespace SZ3; - //h5repack -f UD=32024,0 /home/arham23/Software/SZ3/test/testfloat_8_8_128.dat.h5 tf_8_8_128.dat.sz.h5 -//load from "sz3.config" in local directory if 1 else use default values or cd values - -#define SZ3_CONFIG_PATH "SZ3_CONFIG_PATH" -SZ3::Config sz3_conf; -bool sz3_conf_loaded = false; -int MAX_CHUNK_SIZE = INT_MAX; - //filter definition const H5Z_class2_t H5Z_SZ3[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - (H5Z_filter_t) H5Z_FILTER_SZ3, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ - "SZ3 compressor/decompressor for floating-point data.", /* Filter name for debugging */ - NULL, /* The "can apply" callback */ - H5Z_sz3_set_local, /* The "set local" callback */ - (H5Z_func_t) H5Z_filter_sz3, /* The actual filter function */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t) H5Z_FILTER_SZ3, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "SZ3 compressor/decompressor for floating-point data.", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + H5Z_sz3_set_local, /* The "set local" callback */ + (H5Z_func_t) H5Z_filter_sz3, /* The actual filter function */ } }; @@ -47,1042 +29,182 @@ const void *H5PLget_plugin_info(void) { return H5Z_SZ3; } -/*FILTER FUNCTIONS*/ - -/** - * to be used in compression, and to be called outside H5Z_filter_sz(). - * */ - -void SZ_refreshDimForCdArray(int dataType, size_t old_cd_nelmts, unsigned int *old_cd_values, size_t *new_cd_nelmts, unsigned int **new_cd_values, - size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - unsigned char bytes[8] = {0}; - *new_cd_values = (unsigned int *) malloc(sizeof(unsigned int) * 16); - memset(*new_cd_values, 0, sizeof(unsigned int) * 16); - - //correct dimension if needed - size_t _r[5]; - filterDimension(r5, r4, r3, r2, r1, _r); - size_t _r5 = _r[4]; - size_t _r4 = _r[3]; - size_t _r3 = _r[2]; - size_t _r2 = _r[1]; - size_t _r1 = _r[0]; - - int i = 0; - int oldDim = computeDimension(r5, r4, r3, r2, r1); - int newDim = computeDimension(_r5, _r4, _r3, _r2, _r1); - (*new_cd_values)[0] = newDim; - (*new_cd_values)[1] = dataType; - - - switch (newDim) { - case 1: - longToBytes_bigEndian(bytes, (uint64_t) r1); - (*new_cd_values)[2] = bytesToInt_bigEndian(bytes); - (*new_cd_values)[3] = bytesToInt_bigEndian(&bytes[4]); - if (old_cd_nelmts == 0) - *new_cd_nelmts = 4; - else { - (*new_cd_values)[4] = old_cd_values[0]; - (*new_cd_values)[5] = old_cd_values[1]; - (*new_cd_values)[6] = old_cd_values[2]; - (*new_cd_values)[7] = old_cd_values[3]; - (*new_cd_values)[8] = old_cd_values[4]; - (*new_cd_values)[9] = old_cd_values[5]; - (*new_cd_values)[10] = old_cd_values[6]; - (*new_cd_values)[11] = old_cd_values[7]; - (*new_cd_values)[12] = old_cd_values[8]; - *new_cd_nelmts = 13; - } - break; - case 2: - (*new_cd_values)[2] = (unsigned int) _r2; - (*new_cd_values)[3] = (unsigned int) _r1; - if (old_cd_nelmts == 0) - *new_cd_nelmts = 4; - else { - (*new_cd_values)[4] = old_cd_values[0]; - (*new_cd_values)[5] = old_cd_values[1]; - (*new_cd_values)[6] = old_cd_values[2]; - (*new_cd_values)[7] = old_cd_values[3]; - (*new_cd_values)[8] = old_cd_values[4]; - (*new_cd_values)[9] = old_cd_values[5]; - (*new_cd_values)[10] = old_cd_values[6]; - (*new_cd_values)[11] = old_cd_values[7]; - (*new_cd_values)[12] = old_cd_values[8]; - *new_cd_nelmts = 13; - } - break; - case 3: - (*new_cd_values)[2] = (unsigned int) _r3; - (*new_cd_values)[3] = (unsigned int) _r2; - (*new_cd_values)[4] = (unsigned int) _r1; - if (old_cd_nelmts == 0) - *new_cd_nelmts = 5; - else { - (*new_cd_values)[5] = old_cd_values[0]; - (*new_cd_values)[6] = old_cd_values[1]; - (*new_cd_values)[7] = old_cd_values[2]; - (*new_cd_values)[8] = old_cd_values[3]; - (*new_cd_values)[9] = old_cd_values[4]; - (*new_cd_values)[10] = old_cd_values[5]; - (*new_cd_values)[11] = old_cd_values[6]; - (*new_cd_values)[12] = old_cd_values[7]; - (*new_cd_values)[13] = old_cd_values[8]; - *new_cd_nelmts = 14; - } - break; - case 4: - (*new_cd_values)[2] = (unsigned int) _r4; - (*new_cd_values)[3] = (unsigned int) _r3; - (*new_cd_values)[4] = (unsigned int) _r2; - (*new_cd_values)[5] = (unsigned int) _r1; - if (old_cd_nelmts == 0) - *new_cd_nelmts = 6; - else { - (*new_cd_values)[6] = old_cd_values[0]; - (*new_cd_values)[7] = old_cd_values[1]; - (*new_cd_values)[8] = old_cd_values[2]; - (*new_cd_values)[9] = old_cd_values[3]; - (*new_cd_values)[10] = old_cd_values[4]; - (*new_cd_values)[11] = old_cd_values[5]; - (*new_cd_values)[12] = old_cd_values[6]; - (*new_cd_values)[13] = old_cd_values[7]; - (*new_cd_values)[14] = old_cd_values[8]; - *new_cd_nelmts = 15; - break; - } - default: - (*new_cd_values)[2] = (unsigned int) _r5; - (*new_cd_values)[3] = (unsigned int) _r4; - (*new_cd_values)[4] = (unsigned int) _r3; - (*new_cd_values)[5] = (unsigned int) _r2; - (*new_cd_values)[6] = (unsigned int) _r1; - if (old_cd_nelmts == 0) - *new_cd_nelmts = 7; - else { - (*new_cd_values)[7] = old_cd_values[0]; - (*new_cd_values)[8] = old_cd_values[1]; - (*new_cd_values)[9] = old_cd_values[2]; - (*new_cd_values)[10] = old_cd_values[3]; - (*new_cd_values)[11] = old_cd_values[4]; - (*new_cd_values)[12] = old_cd_values[5]; - (*new_cd_values)[13] = old_cd_values[6]; - (*new_cd_values)[14] = old_cd_values[7]; - (*new_cd_values)[15] = old_cd_values[8]; - *new_cd_nelmts = 16; - } +herr_t set_SZ3_conf_to_H5(const hid_t propertyList, SZ3::Config &conf) { + static char const *_funcname_ = "set_SZ3_conf_to_H5"; + + //save conf into cd_values + size_t cd_nelmts = std::ceil(conf.size_est() / 1.0 / sizeof(int)); + std::vector cd_values(cd_nelmts, 0); + auto buffer = (unsigned char *) (cd_values.data()); + + conf.save(buffer); + auto szfilter = H5Pget_filter_by_id(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, NULL, NULL, 0, NULL, NULL); //check if filter is set + if (0 > szfilter) { //filter not set, set filter. Notice that calling H5Pset_filter twice with the same filter id will cause unexpected errors for decompression + if (0 > H5Pset_filter(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values.data())) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "failed to modify cd_values"); + } + } else { // filter already set, update filter + if (0 > H5Pmodify_filter(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values.data())) + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "failed to modify cd_values"); } -} - - -void -SZ_errConfigToCdArray(size_t *cd_nelmts, unsigned int **cd_values, int error_bound_mode, double abs_error, double rel_error, double l2normErrorBound, - double psnr) { - *cd_values = (unsigned int *) malloc(sizeof(unsigned int) * 9); - int k = 0; - (*cd_values)[k++] = error_bound_mode; - unsigned char b[8]; - doubleToBytes(b, abs_error); - (*cd_values)[k++] = bytesToInt32_bigEndian(b); - (*cd_values)[k++] = bytesToInt32_bigEndian(b + 4); - doubleToBytes(b, rel_error); - (*cd_values)[k++] = bytesToInt32_bigEndian(b); - (*cd_values)[k++] = bytesToInt32_bigEndian(b + 4); - doubleToBytes(b, l2normErrorBound); - (*cd_values)[k++] = bytesToInt32_bigEndian(b); - (*cd_values)[k++] = bytesToInt32_bigEndian(b + 4); - doubleToBytes(b, psnr); - (*cd_values)[k++] = bytesToInt32_bigEndian(b); - (*cd_values)[k++] = bytesToInt32_bigEndian(b + 4); - *cd_nelmts = k; + + return (herr_t) 1; +} + +herr_t get_SZ3_conf_from_H5(const hid_t propertyList, SZ3::Config &conf) { + static char const *_funcname_ = "get_SZ3_conf_from_H5"; + + size_t cd_nelmts = std::ceil(conf.size_est() / 1.0 / sizeof(int)); + std::vector cd_values(cd_nelmts, 0); + + //read cd_values from HDF5 + //note that cd_nelmts must be non-zero, otherwise, cd_values cannot be filled. + if (0 > H5Pget_filter_by_id(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, &cd_nelmts, cd_values.data(), 0, NULL, NULL)) + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "unable to get current SZ cd_values"); + + //load cd_values into config + if (cd_nelmts != 0) { + auto buffer = (const unsigned char *) (cd_values.data()); + conf.load(buffer); + } + return (herr_t) 1; } static herr_t H5Z_sz3_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id) { - - //printf("get into H5Z_sz3_set_local\n"); - detectSysEndianType(); - + + printf("start H5Z_sz3_set_local\n"); + //printf("start in H5Z_sz3_set_local, dcpl_id = %d\n", dcpl_id); static char const *_funcname_ = "H5Z_sz3_set_local"; - size_t r5 = 0, r4 = 0, r3 = 0, r2 = 0, r1 = 0, dsize; - - int i, ndims, ndims_used = 0; - hsize_t dims[H5S_MAX_RANK], dims_used[5] = {0, 0, 0, 0, 0}; - herr_t retval = 0; + + // herr_t ret = H5Zregister(H5Z_SZ3); + + SZ3::Config conf; + get_SZ3_conf_from_H5(dcpl_id, conf); + + //read datatype and dims from HDF5 H5T_class_t dclass; - H5T_sign_t dsign; - unsigned int flags = 0; - size_t mem_cd_nelmts = 9, cd_nelmts = 0; - unsigned int mem_cd_values[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - - //H5Z_FILTER_SZ - //note that mem_cd_nelmts must be non-zero, otherwise, mem_cd_values cannot be filled. - if (0 > H5Pget_filter_by_id(dcpl_id, H5Z_FILTER_SZ3, &flags, &mem_cd_nelmts, mem_cd_values, 0, NULL, NULL)) - H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "unable to get current SZ cd_values"); - - //set default value for error bound - sz3_conf.errorBoundMode = EB_ABS; - sz3_conf.absErrorBound = 1e-3; - if (!sz3_conf_loaded) { - if (const char *conf_file = std::getenv(SZ3_CONFIG_PATH)) { - sz3_conf.loadcfg(conf_file); - sz3_conf_loaded = true; - } - } -// if (mem_cd_nelmts == 0) //this means that the error information is missing from the cd_values -// { -// //printf("mem_cd_nelmets is 0, so let's try using sz3.config to load error configuration....\n"); -// std::ifstream f(CONFIG_PATH); -// if (f.good()) { -// printf("sz3.config found!\n"); -// sz3_conf_loaded = 1; -// } else -// printf("sz3.config not found, using default parameters\n"); -// f.close(); -// } else //this means that the error information is included in the cd_values -// { -// sz3_conf_loaded = 0; -// //printf("mem_cd_nelmets is non-zero, so let's use the parameters set through cd_values.....\n"); -// } - herr_t ret = H5Zregister(H5Z_SZ3); - - int dataType = SZ_FLOAT; - - //printf("DC\n"); if (0 > (dclass = H5Tget_class(type_id))) H5Z_SZ_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a datatype"); - - //printf("DS\n"); + + size_t dsize; if (0 == (dsize = H5Tget_size(type_id))) H5Z_SZ_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "size is smaller than 0!"); - - //printf("ND\n"); - if (0 > (ndims = H5Sget_simple_extent_dims(chunk_space_id, dims, 0))) + + int ndims; + hsize_t dims_all[H5S_MAX_RANK]; + if (0 > (ndims = H5Sget_simple_extent_dims(chunk_space_id, dims_all, 0))) H5Z_SZ_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a data space"); - - for (i = 0; i < ndims; i++) - dims_used[i] = dims[i]; - - - //printf("NDIM: %i\n", ndims); - //printf("N_USE: %i\n", ndims_used); - //printf("DCLASS: %i\n", dclass); - //printf("DSIZE: %zu\n", dsize); - - //for(i = 0; i < ndims_used; i++){ - // printf("DIMS[%i] : %zu\n", i, dims_used[i]); - //} - //printf("\nDCEQ\n"); - + std::vector dims(dims_all, dims_all + ndims); + + //update conf with datatype + conf.dataType = SZ_FLOAT; if (dclass == H5T_FLOAT) - dataType = dsize == 4 ? SZ_FLOAT : SZ_DOUBLE; + conf.dataType = dsize == 4 ? SZ_FLOAT : SZ_DOUBLE; else if (dclass == H5T_INTEGER) { + H5T_sign_t dsign; if (0 > (dsign = H5Tget_sign(type_id))) H5Z_SZ_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "Error in calling H5Tget_sign(type_id)...."); if (dsign == H5T_SGN_NONE) //unsigned { switch (dsize) { - case 1: - dataType = SZ_UINT8; + case 1:conf.dataType = SZ_UINT8; break; - case 2: - dataType = SZ_UINT16; + case 2:conf.dataType = SZ_UINT16; break; - case 4: - dataType = SZ_UINT32; + case 4:conf.dataType = SZ_UINT32; break; - case 8: - dataType = SZ_UINT64; + case 8:conf.dataType = SZ_UINT64; break; } } else { switch (dsize) { - case 1: - dataType = SZ_INT8; + case 1:conf.dataType = SZ_INT8; break; - case 2: - dataType = SZ_INT16; + case 2:conf.dataType = SZ_INT16; break; - case 4: - dataType = SZ_INT32; + case 4:conf.dataType = SZ_INT32; break; - case 8: - dataType = SZ_INT64; + case 8:conf.dataType = SZ_INT64; break; } } } else { H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, 0, "datatype class must be H5T_FLOAT or H5T_INTEGER"); } - - unsigned int *cd_values = NULL; - if (mem_cd_nelmts != 0 && mem_cd_nelmts != 9) { - H5Epush(H5E_DEFAULT, __FILE__, "H5Z_sz3_set_local", __LINE__, H5E_ERR_CLS, H5E_ARGS, H5E_BADVALUE, - "Wrong number of cd_values: The new version has 9 integer elements in cd_values. Please check 'test/print_h5repack_args' to get the correct cd_values."); - H5Eprint(H5E_DEFAULT, stderr); - return -1; + + //update conf with dims + conf.setDims(std::begin(dims), std::end(dims)); + + set_SZ3_conf_to_H5(dcpl_id, conf); + + return (herr_t) 1; +} + +template +void process_data(SZ3::Config &conf, void **buf, size_t *buf_size, size_t nbytes, bool is_decompress) { + if (is_decompress) { + T *processedData = (T *) malloc(conf.num * sizeof(T)); + SZ_decompress(conf, (char *) *buf, nbytes, processedData); + free(*buf); + *buf = processedData; + *buf_size = conf.num * sizeof(T); + } else { + size_t cmpCap = sizeof(T) * conf.num; + char *cmpData = (char *) malloc(cmpCap); + *buf_size = SZ_compress(conf, (T *) *buf, cmpData, cmpCap); + free(*buf); + *buf = cmpData; } - SZ_refreshDimForCdArray(dataType, mem_cd_nelmts, mem_cd_values, &cd_nelmts, &cd_values, dims_used[4], dims_used[3], dims_used[2], dims_used[1], - dims_used[0]); - - /* Now, update cd_values for the filter */ - if (0 > H5Pmodify_filter(dcpl_id, H5Z_FILTER_SZ3, flags, cd_nelmts, cd_values)) - H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "failed to modify cd_values"); - - free(cd_values); - - retval = 1; - done: - return retval; } - +/** + * https://docs.hdfgroup.org/hdf5/v1_14/_f_i_l_t_e_r.html + * The flags, cd_nelmts, and cd_values are the same as for the H5Pset_filter() function with the additional flag H5Z_FLAG_REVERSE which is set when the filter is called as part of the input pipeline. + * The input buffer is pointed to by *buf and has a total size of *buf_size bytes but only nbytes are valid data. + * The filter should perform the transformation in place if possible and return the number of valid bytes or zero for failure. + * If the transformation cannot be done in place then the filter should allocate a new buffer with malloc() and assign it to *buf, assigning the allocated size of that buffer to *buf_size. + * The old buffer should be freed by calling free(). + */ static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) { - //printf("get into H5Z_filter_sz3\n"); - size_t r1 = 0, r2 = 0, r3 = 0, r4 = 0, r5 = 0; - int dimSize = 0, dataType = 0; - + printf("start H5Z_filter_sz3\n"); + if (cd_nelmts == 0) //this is special data such as string, which should not be treated as values. return nbytes; - - int withErrInfo = checkCDValuesWithErrors(cd_nelmts, cd_values); - int error_mode = 0; -// int cmp_algo = 1; -// int interp_algo = 1; - double abs_error = 0, rel_error = 0, l2norm_error = 0, psnr = 0; - if (withErrInfo) - SZ_cdArrayToMetaDataErr(cd_nelmts, cd_values, &dimSize, &dataType, &r5, &r4, &r3, &r2, &r1, &error_mode, &abs_error, &rel_error, - &l2norm_error, &psnr); - else - SZ_cdArrayToMetaData(cd_nelmts, cd_values, &dimSize, &dataType, &r5, &r4, &r3, &r2, &r1); - - /*int i=0; - for(i=0;i dims; - if (r2 == 0) { - dims = {r1}; - } else if (r3 == 0) { - dims = {r2, r1}; - } else if (r4 == 0) { - dims = {r3, r2, r1}; - } else if (r5 == 0) { - dims = {r4, r3, r2, r1}; - } else { - dims = {r5, r4, r3, r2, r1}; - } - conf.setDims(dims.begin(), dims.end()); - - - - //if config file found and no user defined params, read the config file - if (withErrInfo) { - if (error_mode < 0 || error_mode > 5) { - printf("Invalid error mode: %i, error mode should be in [0,5]", error_mode); - exit(0); - } - conf.errorBoundMode = error_mode; - conf.absErrorBound = abs_error; - conf.relErrorBound = rel_error; - conf.l2normErrorBound = l2norm_error; - conf.psnrErrorBound = psnr; - } - //printf("PARAMS: mode|%i, abs_eb|%f, rel_eb|%f, l2_eb|%f, psnr_eb|%f\n", error_mode, abs_error, rel_error, l2norm_error, psnr); - - size_t outSize = 0; - char *compressedData = NULL; - - - switch (dataType) { - case SZ_FLOAT: //FLOAT - { - compressedData = SZ_compress(conf, (float *) *buf, outSize); - break; - } - - case SZ_DOUBLE: //DOUBLE - { - compressedData = SZ_compress(conf, (double *) *buf, outSize); - break; - } - - case SZ_INT8: //INT 8 - { - compressedData = SZ_compress(conf, (int8_t *) *buf, outSize); - break; - } - - case SZ_UINT8: //UINT 8 - { - compressedData = SZ_compress(conf, (uint8_t *) *buf, outSize); - break; - } - - case SZ_INT16: //INT 16 - { - compressedData = SZ_compress(conf, (int16_t *) *buf, outSize); - break; - } - - case SZ_UINT16: //UINT 16 - { - compressedData = SZ_compress(conf, (uint16_t *) *buf, outSize); - break; - } - - case SZ_INT32: //INT 32 - { - compressedData = SZ_compress(conf, (int32_t *) *buf, outSize); - break; - } - - case SZ_UINT32: //UINT 32 - { - compressedData = SZ_compress(conf, (uint32_t *) *buf, outSize); - break; - } - - case SZ_INT64: //INT 64 - { - compressedData = SZ_compress(conf, (int64_t *) *buf, outSize); - break; - } - - case SZ_UINT64: //UINT 64 - { - compressedData = SZ_compress(conf, (uint64_t *) *buf, outSize); - break; - } - - default: { - printf("Compression Error: Unknown Datatype"); - exit(0); - } - } - - //printf("\nOS: %u \n", outSize); - free(*buf); - *buf = compressedData; - *buf_size = outSize; - - - } - - return *buf_size; -} - -/*HELPER FUNCTIONS*/ -//use to convert HDF5 cd_array to SZ params inside filter -void -SZ_cdArrayToMetaData(size_t cd_nelmts, const unsigned int cd_values[], int *dimSize, int *dataType, size_t *r5, size_t *r4, size_t *r3, size_t *r2, - size_t *r1) { - assert(cd_nelmts >= 4); - unsigned char bytes[8]; - *dimSize = cd_values[0]; - *dataType = cd_values[1]; - - switch (*dimSize) { - case 1: - SZ3::int32ToBytes_bigEndian(bytes, cd_values[2]); - SZ3::int32ToBytes_bigEndian(&bytes[4], cd_values[3]); - if (sizeof(size_t) == 4) - *r1 = (unsigned int) SZ3::bytesToInt64_bigEndian(bytes); - else - *r1 = (uint64_t) SZ3::bytesToInt64_bigEndian(bytes); - *r2 = *r3 = *r4 = *r5 = 0; + + bool is_decompress = flags & H5Z_FLAG_REVERSE; + switch (conf.dataType) { + case SZ_FLOAT: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 2: - *r3 = *r4 = *r5 = 0; - *r2 = cd_values[3]; - *r1 = cd_values[2]; + case SZ_DOUBLE: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 3: - *r4 = *r5 = 0; - *r3 = cd_values[4]; - *r2 = cd_values[3]; - *r1 = cd_values[2]; + case SZ_INT8: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 4: - *r5 = 0; - *r4 = cd_values[5]; - *r3 = cd_values[4]; - *r2 = cd_values[3]; - *r1 = cd_values[2]; + case SZ_UINT8: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - default: - *r5 = cd_values[6]; - *r4 = cd_values[5]; - *r3 = cd_values[4]; - *r2 = cd_values[3]; - *r1 = cd_values[2]; - } -} - -void -SZ_cdArrayToMetaDataErr(size_t cd_nelmts, const unsigned int cd_values[], int *dimSize, int *dataType, size_t *r5, size_t *r4, size_t *r3, size_t *r2, - size_t *r1, int *error_bound_mode, double *abs_error, double *rel_error, double *l2norm_error, double *psnr) { - //get dimension, datatype metadata from cd_values - SZ_cdArrayToMetaData(cd_nelmts, cd_values, dimSize, dataType, r5, r4, r3, r2, r1); - //read in error bound value information - int dim = *dimSize; - int k = dim == 1 ? 4 : dim + 2; - unsigned char b[8]; - int32ToBytes_bigEndian(b, cd_values[k++]); - *error_bound_mode = bytesToInt32_bigEndian(b); - int32ToBytes_bigEndian(b, cd_values[k++]); - int32ToBytes_bigEndian(b + 4, cd_values[k++]); - *abs_error = bytesToDouble(b); - int32ToBytes_bigEndian(b, cd_values[k++]); - int32ToBytes_bigEndian(b + 4, cd_values[k++]); - *rel_error = bytesToDouble(b); - int32ToBytes_bigEndian(b, cd_values[k++]); - int32ToBytes_bigEndian(b + 4, cd_values[k++]); - *l2norm_error = bytesToDouble(b); - int32ToBytes_bigEndian(b, cd_values[k++]); - int32ToBytes_bigEndian(b + 4, cd_values[k++]); - *psnr = bytesToDouble(b); -} - -void SZ_copymetaDataToCdArray(size_t *cd_nelmts, unsigned int *cd_values, int dataType, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - unsigned char bytes[8] = {0}; - uint64_t size; - int dim = computeDimension(r5, r4, r3, r2, r1); - cd_values[0] = dim; - cd_values[1] = dataType; //0: FLOAT ; 1: DOUBLE ; 2,3,4,....: INTEGER.... - - switch (dim) { - case 1: - size = (uint64_t) r1; - SZ3::int64ToBytes_bigEndian(bytes, size); - cd_values[2] = SZ3::bytesToInt32_bigEndian(bytes); - cd_values[3] = SZ3::bytesToInt32_bigEndian(&bytes[4]); - *cd_nelmts = 4; + case SZ_INT16: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 2: - cd_values[2] = (unsigned int) r2; - cd_values[3] = (unsigned int) r1; - *cd_nelmts = 4; + case SZ_UINT16: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 3: - cd_values[2] = (unsigned int) r3; - cd_values[3] = (unsigned int) r2; - cd_values[4] = (unsigned int) r1; - *cd_nelmts = 5; + case SZ_INT32: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 4: - cd_values[2] = (unsigned int) r4; - cd_values[3] = (unsigned int) r3; - cd_values[4] = (unsigned int) r2; - cd_values[5] = (unsigned int) r1; - *cd_nelmts = 6; + case SZ_UINT32: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - default: - cd_values[2] = (unsigned int) r5; - cd_values[3] = (unsigned int) r4; - cd_values[4] = (unsigned int) r3; - cd_values[5] = (unsigned int) r2; - cd_values[6] = (unsigned int) r1; - *cd_nelmts = 7; - } -} - -int checkCDValuesWithErrors(size_t cd_nelmts, const unsigned int cd_values[]) { - int result = 0; //0 means no-error-information-in-cd_values; 1 means cd_values contains error information - int dimSize = cd_values[0]; - //printf("nc_nelmts = %d\n", cd_nelmts); - switch (dimSize) { - case 1: - if (cd_nelmts > 4) - result = 1; - break; - case 2: - if (cd_nelmts > 4) - result = 1; - break; - case 3: - if (cd_nelmts > 5) - result = 1; + case SZ_INT64: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 4: - if (cd_nelmts > 6) - result = 1; + case SZ_UINT64: process_data(conf, buf, buf_size, nbytes, is_decompress); break; - case 5: - if (cd_nelmts > 7) - result = 1; - break; - } - return result; -} - -size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - size_t dataLength; - if (r1 == 0) { - dataLength = 0; - } else if (r2 == 0) { - dataLength = r1; - } else if (r3 == 0) { - dataLength = r1 * r2; - } else if (r4 == 0) { - dataLength = r1 * r2 * r3; - } else if (r5 == 0) { - dataLength = r1 * r2 * r3 * r4; - } else { - dataLength = r1 * r2 * r3 * r4 * r5; + default: std::cerr << (is_decompress ? "Decompression" : "Compression") << " Error: Unknown Datatype" << std::endl; + std::exit(EXIT_FAILURE); } - return dataLength; -} - -int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - int dimension; - if (r1 == 0) { - dimension = 0; - } else if (r2 == 0) { - dimension = 1; - } else if (r3 == 0) { - dimension = 2; - } else if (r4 == 0) { - dimension = 3; - } else if (r5 == 0) { - dimension = 4; - } else { - dimension = 5; - } - return dimension; -} - -void init_dims_chunk(int dim, hsize_t dims[5], hsize_t chunk[5], size_t nbEle, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - switch (dim) { - case 1: - dims[0] = r1; - if (nbEle <= MAX_CHUNK_SIZE) //2^32-1 - chunk[0] = r1; - else - chunk[0] = 2147483648;//2^31 - break; - case 2: - dims[0] = r2; - dims[1] = r1; - if (nbEle <= MAX_CHUNK_SIZE) //2^32-1 - { - chunk[0] = r2; - chunk[1] = r1; - } else { - printf("Error: size is too big!\n"); - exit(0); - } - break; - case 3: - dims[0] = r3; - dims[1] = r2; - dims[2] = r1; - if (nbEle <= MAX_CHUNK_SIZE) //2^32-1 - { - chunk[0] = r3; - chunk[1] = r2; - chunk[2] = r1; - } else { - printf("Error: size is too big!\n"); - exit(0); - } - break; - case 4: - dims[0] = r4; - dims[1] = r3; - dims[2] = r2; - dims[3] = r1; - if (nbEle <= MAX_CHUNK_SIZE) //2^32-1 - { - chunk[0] = r4; - chunk[1] = r3; - chunk[2] = r2; - chunk[3] = r1; - } else { - printf("Error: size is too big!\n"); - exit(0); - } - break; - default: - dims[0] = r5; - dims[1] = r4; - dims[2] = r3; - dims[3] = r2; - dims[4] = r1; - if (nbEle <= MAX_CHUNK_SIZE) //2^32-1 - { - chunk[0] = r5; - chunk[1] = r4; - chunk[2] = r3; - chunk[3] = r2; - chunk[4] = r1; - } else { - printf("Error: size is too big!\n"); - exit(0); - } - } -} - -//detect sys endian type -inline void detectSysEndianType() { - //get sys endian type - int x_temp = 1; - char *y_temp = (char *) &x_temp; - - if (*y_temp == 1) - sysEndianType = LITTLE_ENDIAN_SYSTEM; - else //=0 - sysEndianType = BIG_ENDIAN_SYSTEM; -} - -inline void symTransform_8bytes(unsigned char data[8]) { - unsigned char tmp = data[0]; - data[0] = data[7]; - data[7] = tmp; - - tmp = data[1]; - data[1] = data[6]; - data[6] = tmp; - - tmp = data[2]; - data[2] = data[5]; - data[5] = tmp; - - tmp = data[3]; - data[3] = data[4]; - data[4] = tmp; -} - -//the byte to input is in the big-endian format -inline double bytesToDouble(unsigned char *bytes) { - ldouble buf; - memcpy(buf.byte, bytes, 8); - if (sysEndianType == LITTLE_ENDIAN_SYSTEM) - symTransform_8bytes(buf.byte); - return buf.value; -} - -inline void doubleToBytes(unsigned char *b, double num) { - ldouble buf; - buf.value = num; - memcpy(b, buf.byte, 8); - if (sysEndianType == LITTLE_ENDIAN_SYSTEM) - symTransform_8bytes(b); -} - - -/** - * @brief check dimension and correct it if needed - * @return 0 (didn't change dimension) - * 1 (dimension is changed) - * 2 (dimension is problematic) - **/ -int filterDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *correctedDimension) { - int dimensionCorrected = 0; - int dim = computeDimension(r5, r4, r3, r2, r1); - correctedDimension[0] = r1; - correctedDimension[1] = r2; - correctedDimension[2] = r3; - correctedDimension[3] = r4; - correctedDimension[4] = r5; - size_t *c = correctedDimension; - if (dim == 1) { - if (r1 < 1) - return 2; - } else if (dim == 2) { - if (r2 == 1) { - c[1] = 0; - dimensionCorrected = 1; - } - if (r1 == 1) //remove this dimension - { - c[0] = c[1]; - c[1] = c[2]; - dimensionCorrected = 1; - } - } else if (dim == 3) { - if (r3 == 1) { - c[2] = 0; - dimensionCorrected = 1; - } - if (r2 == 1) { - c[1] = c[2]; - c[2] = c[3]; - dimensionCorrected = 1; - } - if (r1 == 1) { - c[0] = c[1]; - c[1] = c[2]; - c[2] = c[3]; - dimensionCorrected = 1; - } - } else if (dim == 4) { - if (r4 == 1) { - c[3] = 0; - dimensionCorrected = 1; - } - if (r3 == 1) { - c[2] = c[3]; - c[3] = c[4]; - dimensionCorrected = 1; - } - if (r2 == 1) { - c[1] = c[2]; - c[2] = c[3]; - c[3] = c[4]; - dimensionCorrected = 1; - } - if (r1 == 1) { - c[0] = c[1]; - c[1] = c[2]; - c[2] = c[3]; - c[3] = c[4]; - dimensionCorrected = 1; - } - } else if (dim == 5) { - if (r5 == 1) { - c[4] = 0; - dimensionCorrected = 1; - } - if (r4 == 1) { - c[3] = c[4]; - c[4] = 0; - dimensionCorrected = 1; - } - if (r3 == 1) { - c[2] = c[3]; - c[3] = c[4]; - c[4] = 0; - dimensionCorrected = 1; - } - if (r2 == 1) { - c[1] = c[2]; - c[2] = c[3]; - c[3] = c[4]; - c[4] = 0; - dimensionCorrected = 1; - } - if (r1 == 1) { - c[0] = c[1]; - c[1] = c[2]; - c[2] = c[3]; - c[3] = c[4]; - c[4] = 0; - dimensionCorrected = 1; - } - } - - return dimensionCorrected; - -} - -inline void longToBytes_bigEndian(unsigned char *b, uint64_t num) { - b[0] = (unsigned char) (num >> 56); - b[1] = (unsigned char) (num >> 48); - b[2] = (unsigned char) (num >> 40); - b[3] = (unsigned char) (num >> 32); - b[4] = (unsigned char) (num >> 24); - b[5] = (unsigned char) (num >> 16); - b[6] = (unsigned char) (num >> 8); - b[7] = (unsigned char) (num); -// if(dataEndianType==LITTLE_ENDIAN_DATA) -// symTransform_8bytes(*b); -} - -inline int bytesToInt_bigEndian(unsigned char *bytes) { - int temp = 0; - int res = 0; - - res <<= 8; - temp = bytes[0] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[1] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[2] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[3] & 0xff; - res |= temp; - - return res; -} - -/** - * @endianType: refers to the endian_type of unsigned char* b. - * */ -inline int64_t bytesToLong_bigEndian(unsigned char *b) { - int64_t temp = 0; - int64_t res = 0; - - res <<= 8; - temp = b[0] & 0xff; - res |= temp; - - res <<= 8; - temp = b[1] & 0xff; - res |= temp; - - res <<= 8; - temp = b[2] & 0xff; - res |= temp; - - res <<= 8; - temp = b[3] & 0xff; - res |= temp; - - res <<= 8; - temp = b[4] & 0xff; - res |= temp; - - res <<= 8; - temp = b[5] & 0xff; - res |= temp; - - res <<= 8; - temp = b[6] & 0xff; - res |= temp; - - res <<= 8; - temp = b[7] & 0xff; - res |= temp; - - return res; -} + return *buf_size; +} \ No newline at end of file diff --git a/tools/H5Z-SZ3/test/CMakeLists.txt b/tools/H5Z-SZ3/test/CMakeLists.txt index 8a765486..e4cfcd01 100644 --- a/tools/H5Z-SZ3/test/CMakeLists.txt +++ b/tools/H5Z-SZ3/test/CMakeLists.txt @@ -4,7 +4,6 @@ function(build_hdf5_test) target_link_libraries(${test_name} PUBLIC SZ3 hdf5sz3) endfunction(build_hdf5_test) -build_hdf5_test(print_h5repack_args.cpp) build_hdf5_test(sz3ToHDF5.cpp) build_hdf5_test(dsz3FromHDF5.cpp) build_hdf5_test(convertBinToHDF5.cpp) diff --git a/tools/H5Z-SZ3/test/convertBinToHDF5.cpp b/tools/H5Z-SZ3/test/convertBinToHDF5.cpp index d628f707..5c708866 100644 --- a/tools/H5Z-SZ3/test/convertBinToHDF5.cpp +++ b/tools/H5Z-SZ3/test/convertBinToHDF5.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -7,206 +7,206 @@ int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - int dimension; - if(r1==0) - { - dimension = 0; - } - else if(r2==0) - { - dimension = 1; - } - else if(r3==0) - { - dimension = 2; - } - else if(r4==0) - { - dimension = 3; - } - else if(r5==0) - { - dimension = 4; - } - else - { - dimension = 5; - } - return dimension; + int dimension; + if(r1==0) + { + dimension = 0; + } + else if(r2==0) + { + dimension = 1; + } + else if(r3==0) + { + dimension = 2; + } + else if(r4==0) + { + dimension = 3; + } + else if(r5==0) + { + dimension = 4; + } + else + { + dimension = 5; + } + return dimension; } size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - size_t dataLength; - if(r1==0) - { - dataLength = 0; - } - else if(r2==0) - { - dataLength = r1; - } - else if(r3==0) - { - dataLength = r1*r2; - } - else if(r4==0) - { - dataLength = r1*r2*r3; - } - else if(r5==0) - { - dataLength = r1*r2*r3*r4; - } - else - { - dataLength = r1*r2*r3*r4*r5; - } - return dataLength; + size_t dataLength; + if(r1==0) + { + dataLength = 0; + } + else if(r2==0) + { + dataLength = r1; + } + else if(r3==0) + { + dataLength = r1*r2; + } + else if(r4==0) + { + dataLength = r1*r2*r3; + } + else if(r5==0) + { + dataLength = r1*r2*r3*r4; + } + else + { + dataLength = r1*r2*r3*r4*r5; + } + return dataLength; } int main(int argc, char* argv[]) { - - hid_t file_id, dataset_id, dataspace_id; /* identifiers */ - hsize_t dims[3]; - herr_t status; - size_t r1 = 0, r2 = 0, r3 = 0, r4 = 0, r5 = 0; - - char varName[100], infile[100], outfile[100], database[100]; - char datatype[100]; - - if(argc<5) - { - printf("Usage: convertBinToHDF5 [datatype] [varName] [infile] [r1, r2, r3, ....]\n"); - printf("[datatype] options: -f float32\t-d float64\t-i32 int32\t-i64 int64\n"); - exit(0); - } - - - - strcpy(datatype, argv[1]); - strcpy(varName, argv[2]); - strcpy(infile, argv[3]); - - if(argc>=5) - r1 = atoi(argv[4]); //8 - if(argc>=6) - r2 = atoi(argv[5]); //8 - if(argc>=7) - r3 = atoi(argv[6]); //128 - if(argc>=8) - r4 = atoi(argv[7]); - if(argc>=9) - r5 = atoi(argv[8]); - - int dim = computeDimension(r5, r4, r3, r2, r1); - size_t nbEle = computeDataLength(r5, r4, r3, r2, r1); - - snprintf(outfile, 100, "%s.h5", infile); - - /* Create a new file using default properties. */ - file_id = H5Fcreate(outfile, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - - /* Create the data space for the dataset. */ - if(dim==1) - dims[0] = r1; - else if(dim==2) - { - dims[0] = r2; - dims[1] = r1; - } - else if(dim==3) - { - dims[0] = r3; - dims[1] = r2; - dims[2] = r1; - } - else - { - printf("Error: wrong dimension\n"); - exit(0); - } - dataspace_id = H5Screate_simple(dim, dims, NULL); - - /* Create the dataset. */ - snprintf(database, 100, "/%s", varName); - - if(strcmp(datatype, "-f") == 0){ - - FILE *f; - f = fopen(infile, "rb"); - float *data = (float*)malloc(nbEle*sizeof(float)); - fread(data, sizeof(float), nbEle, f); - fclose(f); - - dataset_id = H5Dcreate2(file_id, database, H5T_IEEE_F32LE, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - status = H5Dwrite(dataset_id, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data); - - hsize_t di_ms[H5S_MAX_RANK]; - int ndims; - ndims = H5Sget_simple_extent_dims(dataspace_id, di_ms, 0); - printf("NDIMS: %i\n", ndims); - - } - else if(strcmp(datatype, "-d") == 0){ - FILE *f; - f = fopen(infile, "rb"); - double *data = (double*)malloc(nbEle*sizeof(double)); - fread(data, sizeof(double), nbEle, f); - fclose(f); - - dataset_id = H5Dcreate2(file_id, database, H5T_IEEE_F64LE, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - status = H5Dwrite(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data); - - } - else if(strcmp(datatype, "-i32") == 0){ - FILE *f; - f = fopen(infile, "rb"); - int *data = (int*)malloc(nbEle*sizeof(int)); - fread(data, sizeof(int), nbEle, f); - fclose(f); - - dataset_id = H5Dcreate2(file_id, database, H5T_STD_I32LE, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - status = H5Dwrite(dataset_id, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data); - - } - else if(strcmp(datatype, "-i64") == 0){ - FILE *f; - f = fopen(infile, "rb"); - int64_t *data = (int64_t*)malloc(nbEle*sizeof(int64_t)); - fread(data, sizeof(int64_t), nbEle, f); - fclose(f); + + hid_t file_id, dataset_id, dataspace_id; /* identifiers */ + hsize_t dims[3]; + herr_t status; + size_t r1 = 0, r2 = 0, r3 = 0, r4 = 0, r5 = 0; + + char varName[100], infile[100], outfile[100], database[100]; + char datatype[100]; + + if(argc<5) + { + printf("Usage: convertBinToHDF5 [datatype] [varName] [infile] [r1, r2, r3, ....]\n"); + printf("[datatype] options: -f float32\t-d float64\t-i32 int32\t-i64 int64\n"); + exit(0); + } + + + + strcpy(datatype, argv[1]); + strcpy(varName, argv[2]); + strcpy(infile, argv[3]); + + if(argc>=5) + r1 = atoi(argv[4]); //8 + if(argc>=6) + r2 = atoi(argv[5]); //8 + if(argc>=7) + r3 = atoi(argv[6]); //128 + if(argc>=8) + r4 = atoi(argv[7]); + if(argc>=9) + r5 = atoi(argv[8]); + + int dim = computeDimension(r5, r4, r3, r2, r1); + size_t nbEle = computeDataLength(r5, r4, r3, r2, r1); + + snprintf(outfile, 100, "%s.h5", infile); + + /* Create a new file using default properties. */ + file_id = H5Fcreate(outfile, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + /* Create the data space for the dataset. */ + if(dim==1) + dims[0] = r1; + else if(dim==2) + { + dims[0] = r2; + dims[1] = r1; + } + else if(dim==3) + { + dims[0] = r3; + dims[1] = r2; + dims[2] = r1; + } + else + { + printf("Error: wrong dimension\n"); + exit(0); + } + dataspace_id = H5Screate_simple(dim, dims, NULL); + + /* Create the dataset. */ + snprintf(database, 100, "/%s", varName); + + if(strcmp(datatype, "-f") == 0){ - dataset_id = H5Dcreate2(file_id, database, H5T_STD_I64LE, dataspace_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - status = H5Dwrite(dataset_id, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, - data); - - } - else{ - printf("Invalid datatype, use ./convertBinToHDF5 to see usage"); - exit(0); - } - - - /* End access to the dataset and release resources used by it. */ - status = H5Dclose(dataset_id); - - /* Terminate access to the data space. */ - status = H5Sclose(dataspace_id); - - /* Close the file. */ - status = H5Fclose(file_id); + FILE *f; + f = fopen(infile, "rb"); + float *data = (float*)malloc(nbEle*sizeof(float)); + fread(data, sizeof(float), nbEle, f); + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_IEEE_F32LE, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data); + + hsize_t di_ms[H5S_MAX_RANK]; + int ndims; + ndims = H5Sget_simple_extent_dims(dataspace_id, di_ms, 0); + printf("NDIMS: %i\n", ndims); + + } + else if(strcmp(datatype, "-d") == 0){ + FILE *f; + f = fopen(infile, "rb"); + double *data = (double*)malloc(nbEle*sizeof(double)); + fread(data, sizeof(double), nbEle, f); + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_IEEE_F64LE, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data); + + } + else if(strcmp(datatype, "-i32") == 0){ + FILE *f; + f = fopen(infile, "rb"); + int *data = (int*)malloc(nbEle*sizeof(int)); + fread(data, sizeof(int), nbEle, f); + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_STD_I32LE, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data); + + } + else if(strcmp(datatype, "-i64") == 0){ + FILE *f; + f = fopen(infile, "rb"); + int64_t *data = (int64_t*)malloc(nbEle*sizeof(int64_t)); + fread(data, sizeof(int64_t), nbEle, f); + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_STD_I64LE, dataspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data); + + } + else{ + printf("Invalid datatype, use ./convertBinToHDF5 to see usage"); + exit(0); + } + + + /* End access to the dataset and release resources used by it. */ + status = H5Dclose(dataset_id); + + /* Terminate access to the data space. */ + status = H5Sclose(dataspace_id); + + /* Close the file. */ + status = H5Fclose(file_id); } diff --git a/tools/H5Z-SZ3/test/dsz3FromHDF5.cpp b/tools/H5Z-SZ3/test/dsz3FromHDF5.cpp index 32ed97ce..ab046e82 100644 --- a/tools/H5Z-SZ3/test/dsz3FromHDF5.cpp +++ b/tools/H5Z-SZ3/test/dsz3FromHDF5.cpp @@ -17,295 +17,269 @@ #define DATASET "testdata_compressed" #define MAX_CHUNK_SIZE 4294967295 //2^32-1 -int main(int argc, char * argv[]) -{ - int dimSize = 0; - size_t r5=0,r4=0,r3=0,r2=0,r1=0,nbEle = 0; - char hdf5FilePath[640], outputFilePath[640]; - hid_t file, dset, dcpl, space_id, dtype; /*Handles*/ - hid_t fid, sid, cpid, idsid; /*Output Handles*/ - H5Z_filter_t filter_id = 0; - herr_t status; - H5T_class_t type_class; - H5T_sign_t dsign; - H5T_order_t dorder; - - htri_t avail; - char filter_name[80]; - unsigned int flags = 0; - size_t nelmts = 0, dsize; - unsigned int values_out[7] = {0,0,0,0,0,0,0}; //at most 7 parameters - - //hold dims - hsize_t dims[H5S_MAX_RANK], dims_used[5] = {0,0,0,0,0}; - int ndims, dim; - - - if(argc < 2) - { - printf("Test case: dszFromHDF5 [hdf5FilePath]\n"); - printf("Example 1: dszFromHDF5 testdata/x86/testfloat_8_8_128.dat.sz3.h5\n"); - printf("Example 2: dszFromHDF5 testdata/x86/testint32_8x8x8.dat.sz3.h5\n"); - exit(0); - } - - snprintf(hdf5FilePath, 640, "%s", argv[1]); - snprintf(outputFilePath, 640, "%s.out.h5", hdf5FilePath); - - /*Open the hdf5 file with SZ-compressed data*/ +int main(int argc, char *argv[]) { + int dimSize = 0; + size_t r5 = 0, r4 = 0, r3 = 0, r2 = 0, r1 = 0, nbEle = 0; + char hdf5FilePath[640], outputFilePath[640]; + hid_t file, dset, dcpl, space_id, dtype; /*Handles*/ + hid_t fid, sid, cpid, idsid; /*Output Handles*/ + H5Z_filter_t filter_id = 0; + herr_t status; + H5T_class_t type_class; + H5T_sign_t dsign; + H5T_order_t dorder; + + htri_t avail; + char filter_name[80]; + unsigned int flags = 0; + size_t nelmts = 0, dsize; + unsigned int values_out[7] = {0, 0, 0, 0, 0, 0, 0}; //at most 7 parameters + + //hold dims + hsize_t dims[H5S_MAX_RANK], dims_used[5] = {0, 0, 0, 0, 0}; + int ndims, dim; + + if (argc < 2) { + printf("Test case: dszFromHDF5 [hdf5FilePath]\n"); + printf("Example 1: dszFromHDF5 testdata/x86/testfloat_8_8_128.dat.sz3.h5\n"); + printf("Example 2: dszFromHDF5 testdata/x86/testint32_8x8x8.dat.sz3.h5\n"); + exit(0); + } + + snprintf(hdf5FilePath, 640, "%s", argv[1]); + snprintf(outputFilePath, 640, "%s.out.h5", hdf5FilePath); + + /*Open the hdf5 file with SZ-compressed data*/ file = H5Fopen(hdf5FilePath, H5F_ACC_RDONLY, H5P_DEFAULT); dset = H5Dopen(file, DATASET, H5P_DEFAULT); /*Retrieve dataset creation property list.*/ dcpl = H5Dget_create_plist(dset); - + + herr_t ret = H5Zregister(H5PLget_plugin_info()); + /*Check that filter is not registered with the library yet*/ - avail = H5Zfilter_avail(H5Z_FILTER_SZ3); - if(!avail) - printf("sz3 filter is not yet available after the H5Pget_filter call.\n"); - else - printf("sz3 filter is available.\n"); - - space_id = H5Dget_space(dset); - nbEle = H5Sget_simple_extent_npoints(space_id); - - if((dtype = H5Dget_type(dset)) < 0) - printf("Error: H5Dget_type(dset) < 0\n"); - - /*Read the data using the default properties.*/ - printf("....Reading SZ3 compressed data .....................\n"); - - if((type_class = H5Tget_class(dtype)) < 0) - { - printf("Error: H5Tget_class<0\n"); - exit(0); - } - if (0 == (dsize = H5Tget_size(dtype))) - { - printf("Error: H5Tget_size==0\n"); - exit(0); - } - - if((dorder = H5Tget_order(dtype)) < 0) - printf("Error: H5Tget_order<0\n"); - - //create output file /*Get Dimensions for writing*/ - - - if (0 > (ndims = H5Sget_simple_extent_dims(space_id, dims, 0))) ERROR(H5Sget_simple_extent_dims); - - int ndims_used = 0; - for(int i = 0; i < ndims; i++){ - if(dims[i] <= 1) continue; - dims_used[ndims_used] = dims[i]; - ndims_used++; - } - - dim = ndims_used; - - - /* create HDF5 file */ - if (0 > (fid = H5Fcreate(outputFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Fcreate); - - /*Create dataspace. Setting maximum size */ - if (0 > (sid = H5Screate_simple(dim, dims, NULL))) ERROR(H5Screate_simple); - - /* setup dataset creation properties */ - if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) ERROR(H5Pcreate); - - - switch (type_class) - { - case H5T_FLOAT: - if (H5Tequal(dtype, H5T_IEEE_F32BE) == 1 || H5Tequal(dtype, H5T_IEEE_F32LE) == 1 - || H5Tequal(dtype, H5T_NATIVE_FLOAT) == 1) - { - printf("data type: float\n"); - float* data = (float*)malloc(sizeof(float)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else //H5T_ORDER_BE - status = H5Dread(dset, H5T_IEEE_F32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - /*Print the first 20 data values to check the correctness.*/ - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%f ", data[i]); - printf("\n"); - - //write out file data - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); - if (0 > H5Dwrite(idsid, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) ERROR(H5Dwrite); - if (0 > H5Dclose(idsid)) ERROR(H5Dclose); - - - free(data); - } - else //64bit: double - { - printf("data type: double\n"); - double* data = (double*)malloc(sizeof(double)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_IEEE_F64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - /*Print the first 10 data values to check the correctness.*/ - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%f ", data[i]); - printf("\n"); - free(data); - } - break; - case H5T_INTEGER: - if (0 > (dsign = H5Tget_sign(dtype))) - { - printf("Error in calling H5Tget_sign(type_id)....\n"); - exit(0); - } - if(dsign == H5T_SGN_NONE) //unsigned - { - if(dsize==1) - { - printf("data type: unsigned char\n"); - unsigned char* data = (unsigned char*)malloc(sizeof(unsigned char)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_U8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_U8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%d ", data[i]); - printf("\n"); - free(data); - } - else if(dsize==2) - { - printf("data type: unsigned short\n"); - unsigned short* data = (unsigned short*)malloc(sizeof(unsigned short)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_U16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_U16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%d ", data[i]); - printf("\n"); - free(data); - } - else if(dsize==4) - { - printf("data type: unsigned int\n"); - unsigned int* data = (unsigned int*)malloc(sizeof(unsigned int)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_U32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_U32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%d ", data[i]); - printf("\n"); - free(data); - } - else if(dsize==8) - { - printf("data type: unsigned long\n"); - uint64_t* data = (uint64_t*)malloc(sizeof(uint64_t)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_U64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_U64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%ld ", data[i]); - printf("\n"); - free(data); - } - } - else - { - if(dsize==1) - { - printf("data type: char\n"); - char *data = (char*)malloc(sizeof(char)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_I8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_I8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%d ", data[i]); - printf("\n"); - free(data); - } - else if(dsize==2) - { - printf("data type: short\n"); - short *data = (short*)malloc(sizeof(short)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_I16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_I16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%d ", data[i]); - printf("\n"); - free(data); - } - else if(dsize==4) - { - printf("data type: int\n"); - int *data = (int*)malloc(sizeof(int)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_I32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%d ", data[i]); - printf("\n"); - free(data); - } - else if(dsize==8) - { - printf("data type: long\n"); - int64_t *data = (int64_t*)malloc(sizeof(int64_t)*nbEle); - if(dorder==H5T_ORDER_LE) - status = H5Dread(dset, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - else - status = H5Dread(dset, H5T_STD_I64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - int i; - printf("reconstructed data = "); - for(i=0;i<20;i++) - printf("%ld ", data[i]); - printf("\n"); - free(data); - } - } - - break; - default: - printf("Error: H5Z-SZ3 supports only float, double or integers.\n"); - exit(1); - } - - //Close reading resources - status = H5Pclose(dcpl); - status = H5Dclose(dset); - status = H5Fclose(file); - - /*Close and release writing resources*/ - if (0 > H5Sclose(sid)) ERROR(H5Sclose); - if (0 > H5Pclose(cpid)) ERROR(H5Pclose); - if (0 > H5Fclose(fid)) ERROR(H5Fclose); - - return 0; + avail = H5Zfilter_avail(H5Z_FILTER_SZ3); + if (!avail) + printf("sz3 filter is not yet available after the H5Pget_filter call.\n"); + else + printf("sz3 filter is available.\n"); + + space_id = H5Dget_space(dset); + nbEle = H5Sget_simple_extent_npoints(space_id); + + if ((dtype = H5Dget_type(dset)) < 0) + printf("Error: H5Dget_type(dset) < 0\n"); + + /*Read the data using the default properties.*/ + printf("....Reading SZ3 compressed data .....................\n"); + + if ((type_class = H5Tget_class(dtype)) < 0) { + printf("Error: H5Tget_class<0\n"); + exit(0); + } + if (0 == (dsize = H5Tget_size(dtype))) { + printf("Error: H5Tget_size==0\n"); + exit(0); + } + + if ((dorder = H5Tget_order(dtype)) < 0) + printf("Error: H5Tget_order<0\n"); + + //create output file /*Get Dimensions for writing*/ + + + if (0 > (ndims = H5Sget_simple_extent_dims(space_id, dims, 0))) ERROR(H5Sget_simple_extent_dims); + + int ndims_used = 0; + for (int i = 0; i < ndims; i++) { + if (dims[i] <= 1) continue; + dims_used[ndims_used] = dims[i]; + ndims_used++; + } + + dim = ndims_used; + + + /* create HDF5 file */ + if (0 > (fid = H5Fcreate(outputFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Fcreate); + + /*Create dataspace. Setting maximum size */ + if (0 > (sid = H5Screate_simple(dim, dims, NULL))) ERROR(H5Screate_simple); + + /* setup dataset creation properties */ + if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) ERROR(H5Pcreate); + + switch (type_class) { + case H5T_FLOAT: + if (H5Tequal(dtype, H5T_IEEE_F32BE) == 1 || H5Tequal(dtype, H5T_IEEE_F32LE) == 1 + || H5Tequal(dtype, H5T_NATIVE_FLOAT) == 1) { + printf("data type: float\n"); + float *data = (float *) malloc(sizeof(float) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else //H5T_ORDER_BE + status = H5Dread(dset, H5T_IEEE_F32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + /*Print the first 20 data values to check the correctness.*/ + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%f ", data[i]); + printf("\n"); + + //write out file data + if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) ERROR(H5Dcreate); + if (0 > H5Dwrite(idsid, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) ERROR(H5Dwrite); + if (0 > H5Dclose(idsid)) ERROR(H5Dclose); + + free(data); + } else //64bit: double + { + printf("data type: double\n"); + double *data = (double *) malloc(sizeof(double) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_IEEE_F64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + /*Print the first 10 data values to check the correctness.*/ + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%f ", data[i]); + printf("\n"); + free(data); + } + break; + case H5T_INTEGER: + if (0 > (dsign = H5Tget_sign(dtype))) { + printf("Error in calling H5Tget_sign(type_id)....\n"); + exit(0); + } + if (dsign == H5T_SGN_NONE) //unsigned + { + if (dsize == 1) { + printf("data type: unsigned char\n"); + unsigned char *data = (unsigned char *) malloc(sizeof(unsigned char) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 2) { + printf("data type: unsigned short\n"); + unsigned short *data = (unsigned short *) malloc(sizeof(unsigned short) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 4) { + printf("data type: unsigned int\n"); + unsigned int *data = (unsigned int *) malloc(sizeof(unsigned int) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 8) { + printf("data type: unsigned long\n"); + uint64_t *data = (uint64_t *) malloc(sizeof(uint64_t) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%llu ", data[i]); + printf("\n"); + free(data); + } + } else { + if (dsize == 1) { + printf("data type: char\n"); + char *data = (char *) malloc(sizeof(char) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 2) { + printf("data type: short\n"); + short *data = (short *) malloc(sizeof(short) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 4) { + printf("data type: int\n"); + int *data = (int *) malloc(sizeof(int) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 8) { + printf("data type: long\n"); + int64_t *data = (int64_t *) malloc(sizeof(int64_t) * nbEle); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + int i; + printf("reconstructed data = "); + for (i = 0; i < 20; i++) + printf("%lld ", data[i]); + printf("\n"); + free(data); + } + } + + break; + default: printf("Error: H5Z-SZ3 supports only float, double or integers.\n"); + exit(1); + } + + //Close reading resources + status = H5Pclose(dcpl); + status = H5Dclose(dset); + status = H5Fclose(file); + + /*Close and release writing resources*/ + if (0 > H5Sclose(sid)) ERROR(H5Sclose); + if (0 > H5Pclose(cpid)) ERROR(H5Pclose); + if (0 > H5Fclose(fid)) ERROR(H5Fclose); + + return 0; } diff --git a/tools/H5Z-SZ3/test/print_h5repack_args.cpp b/tools/H5Z-SZ3/test/print_h5repack_args.cpp deleted file mode 100644 index 3549df2b..00000000 --- a/tools/H5Z-SZ3/test/print_h5repack_args.cpp +++ /dev/null @@ -1,219 +0,0 @@ -#include -#include -#include -#include -#include - -#define ABS 0 -#define REL 1 -#define NORM2 2 -#define PSNR 3 - - -#define LITTLE_ENDIAN_SYSTEM 0 -#define BIG_ENDIAN_SYSTEM 1 -#define LITTLE_ENDIAN_DATA 0 -#define BIG_ENDIAN_DATA 1 - - -int sysEndianType = LITTLE_ENDIAN_SYSTEM; -int dataEndianType = LITTLE_ENDIAN_DATA; - -typedef union ldouble -{ - double value; - uint64_t lvalue; - unsigned char byte[8]; -} ldouble; - -ldouble buf; - -void usage() -{ - printf("Usage: print_h5repack_args \n"); - printf("Options:\n"); - printf(" -M : 10 options as follows. \n"); - printf(" ABS (absolute error bound)\n"); - printf(" REL (value range based error bound, so a.k.a., VR_REL)\n"); - printf(" PSNR (peak signal-to-noise ratio)\n"); - printf(" NORM2 (norm2)\n"); - printf(" -A : specifying absolute error bound\n"); - printf(" -R : specifying relative error bound\n"); - printf(" -N : specifying norm2 error bound\n"); - printf(" -S : specifying PSNR\n"); - printf("* examples: \n"); - printf(" print_h5repack_args -M ABS -A 1E-3 (output: -f UD=32024,0,9,0,1062232653,3539053052,0,0,0,0,0,0)\n"); - printf(" print_h5repack_args -M REL -R 1E-4 (output: -f UD=32024,0,9,1,0,0,1058682594,3944497965,0,0,0,0)\n"); - exit(0); -} - -int bytesToInt32_bigEndian(unsigned char* bytes) -{ - int temp = 0; - int res = 0; - - res <<= 8; - temp = bytes[0] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[1] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[2] & 0xff; - res |= temp; - - res <<= 8; - temp = bytes[3] & 0xff; - res |= temp; - - return res; -} - -void symTransform_8bytes(unsigned char data[8]) -{ - unsigned char tmp = data[0]; - data[0] = data[7]; - data[7] = tmp; - - tmp = data[1]; - data[1] = data[6]; - data[6] = tmp; - - tmp = data[2]; - data[2] = data[5]; - data[5] = tmp; - - tmp = data[3]; - data[3] = data[4]; - data[4] = tmp; -} - -void doubleToBytes(unsigned char *b, double num) -{ - ldouble buf; - buf.value = num; - memcpy(b, buf.byte, 8); - if(sysEndianType==LITTLE_ENDIAN_SYSTEM) - symTransform_8bytes(b); -} - -int main(int argc, char* argv[]) -{ - char* errBoundMode = NULL; - char* absErrorBound = NULL; - char* relErrorBound = NULL; - char* norm2ErrorBound = NULL; - char* psnr_ = NULL; - - if(argc==1) - usage(); - - int i = 0; - - for(i=1;i #include #include +#include + #include "hdf5.h" +#include "H5Cpp.h" #include "H5Z_SZ3.hpp" #include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Config.hpp" +#define LITTLE_ENDIAN_SYSTEM 0 +#define BIG_ENDIAN_SYSTEM 1 +#define LITTLE_ENDIAN_DATA 0 +#define BIG_ENDIAN_DATA 1 int sysEndianType = LITTLE_ENDIAN_SYSTEM; int dataEndianType = LITTLE_ENDIAN_DATA; #define DATASET "testdata_compressed" +int MAX_CHUNK_SIZE = INT_MAX; + +//detect sys endian type +inline void detectSysEndianType() { + //get sys endian type + int x_temp = 1; + char *y_temp = (char *) &x_temp; + + if (*y_temp == 1) + sysEndianType = LITTLE_ENDIAN_SYSTEM; + else //=0 + sysEndianType = BIG_ENDIAN_SYSTEM; +} -using namespace SZ3; +template +void process_data(const SZ3::Config &conf, + const char *oriFilePath, + int dataEndianType, + hid_t fid, + hid_t sid, + hid_t cpid, + const char *datasetName, + hid_t h5TypeLE, + hid_t h5TypeBE) { + T *data = new T[conf.num]; + SZ3::readfile(oriFilePath, conf.num, data); + + std::cout << "original data = "; + for (int i = 0; i < 20; ++i) + std::cout << data[i] << " "; + std::cout << "....\n"; + + hid_t dataset; + hid_t h5Type = (dataEndianType == LITTLE_ENDIAN_DATA) ? h5TypeLE : h5TypeBE; + + if ((dataset = H5Dcreate(fid, datasetName, h5Type, sid, H5P_DEFAULT, cpid, H5P_DEFAULT)) < 0) { + std::cerr << "Error in H5Dcreate\n"; + delete[] data; + exit(EXIT_FAILURE); + } + if (H5Dwrite(dataset, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) { + std::cerr << "Error in H5Dwrite\n"; + delete[] data; + exit(EXIT_FAILURE); + } + + delete[] data; + if (H5Dclose(dataset) < 0) { + std::cerr << "Error in H5Dclose\n"; + exit(EXIT_FAILURE); + } +} int main(int argc, char *argv[]) { - - //(void) helper fn to detect system endian type - //detectSysEndianType(); - //by default sysEndianType and dataEndianType are little endian, can set them manually here - //dataEndianType = BIG_ENDIAN_DATA; - - size_t r5 = 0, r4 = 0, r3 = 0, r2 = 0, r1 = 0; - int cmp_algo, interp_algo; //select compression and interpolation for SZ3 - char outDir[640], oriFilePath[640], outputFilePath[640]; - size_t cd_nelmts=0, nbEle; - unsigned int *cd_values = NULL; - //unsigned int cd_values[7]; - - herr_t status; - htri_t avail; - unsigned filter_config; - - hid_t sid, idsid, cpid, fid; - + + char oriFilePath[640], outputFilePath[640]; + if (argc < 3) { printf("Test case: sz3ToHDF5 [dataType] [srcFilePath] [dimension sizes...]\n"); printf("Example1 : sz3ToHDF5 -f testdata/x86/testfloat_8_8_128.dat 8 8 128\n"); printf("Example 2: sz3ToHDF5 -i32 testdata/x86/testint32_8x8x8.dat 8 8 8\n"); exit(0); } - - //printf("config file = %s\n", argv[2]); - + + std::map dataTypeMap = { + {"-f", SZ_FLOAT}, {"-d", SZ_DOUBLE}, {"-i8", SZ_INT8}, {"-u8", SZ_UINT8}, + {"-i16", SZ_INT16}, {"-u16", SZ_UINT16}, {"-i32", SZ_INT32}, {"-u32", SZ_UINT32}, + {"-i64", SZ_INT64}, {"-u64", SZ_UINT64} + }; + int dataType = 0; - if (strcmp(argv[1], "-f") == 0) - dataType = SZ_FLOAT; - else if (strcmp(argv[1], "-d") == 0) - dataType = SZ_DOUBLE; - else if (strcmp(argv[1], "-i8") == 0) - dataType = SZ_INT8; - else if (strcmp(argv[1], "-u8") == 0) - dataType = SZ_UINT8; - else if (strcmp(argv[1], "-i16") == 0) - dataType = SZ_INT16; - else if (strcmp(argv[1], "-u16") == 0) - dataType = SZ_UINT16; - else if (strcmp(argv[1], "-i32") == 0) - dataType = SZ_INT32; - else if (strcmp(argv[1], "-u32") == 0) - dataType = SZ_UINT32; - else if (strcmp(argv[1], "-i64") == 0) - dataType = SZ_INT64; - else if (strcmp(argv[1], "-u64") == 0) - dataType = SZ_UINT64; - else { - printf("Error: unknown data type in sz3ToHDF5.c!\n"); - exit(0); + auto it = dataTypeMap.find(argv[1]); + if (it != dataTypeMap.end()) { + dataType = it->second; + } else { + std::cerr << "Error: unknown data type in sz3ToHDF5.c!\n"; + return 0; } - - printf("DTYPE: %i", dataType); + snprintf(oriFilePath, 640, "%s", argv[2]); - if (argc >= 4) { - r1 = atoi(argv[3]); //8 - } - if (argc >= 5) { - r2 = atoi(argv[4]); //8 + + std::vector dimensions; + for (int i = 3; i < argc && i < 8; ++i) { + dimensions.push_back(std::atoi(argv[i])); } - if (argc >= 6) { - r3 = atoi(argv[5]); //128 - } - if (argc >= 7) { - r4 = atoi(argv[6]); - } - if (argc >= 8) { - r5 = atoi(argv[7]); - } - - //read in compression and interp algo - //for testing set these here as defaults in config - cmp_algo = 1; - interp_algo = 1; - - //printf("cfgFile=%s\n", cfgFile); + std::reverse(dimensions.begin(), dimensions.end()); // slowest to fastest + snprintf(outputFilePath, 640, "%s.sz3.h5", oriFilePath); - -// printf("argv[1]=%s, dataType=%d\n", argv[1], dataType); - nbEle = computeDataLength(r5, r4, r3, r2, r1); - -// printf("nbEle=%u\n", nbEle); - - //Create cd_values - printf("Dimension sizes: n5=%u, n4=%u, n3=%u, n2=%u, n1=%u\n", r5, r4, r3, r2, r1); - int mode = 0; //0: ABS, 1: REL, ... - SZ_errConfigToCdArray(&cd_nelmts, &cd_values, mode, 0.001, 0.001, 0, - 0); //SZ_FLOAT or SZ_DOUBLE or SZ_INT 100x500x500 : 0, 0, 100, 500, 500, ABS, REL (0.01, 0.01*(max-min), PW_REL (0.01, 5, 6, 7, 8, 9 --> 5*0.01, 6*0.01, ...), PSNR (mean squared error)). - //load_conffile_flag = 0; - // REL - //SZ_metaDataErrToCdArray(&cd_nelmts, &cd_values, dataType, r5, r4, r3, r2, r1, 1, 0.01, 0.01, 0, 0); //SZ_FLOAT or SZ_DOUBLE or SZ_INT 100x500x500 : 0, 0, 100, 500, 500, ABS, REL (0.01, 0.01*(max-min), PW_REL (0.01, 5, 6, 7, 8, 9 --> 5*0.01, 6*0.01, ...), PSNR (mean squared error)). - /*cd_nelmts = 5; - cd_values[0] = 3; - cd_values[1] = 0; - cd_values[2] = 128; - cd_values[3] = 8; - cd_values[4] = 8; - cd_values[5] = 0; - cd_values[6] = 0;*/ - - int i = 0; -// for(i=0;i (fid = H5Fcreate(outputFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) { printf("Error in H5Pcreate"); exit(0); } - + + + //set up SZ configuration + SZ3::Config conf; + // use config.loadcfg("path_to_sz3_conf") to load configuration from a file if needed + // setting up data related attributes (data type, dims, etc.) is not necessary, as they will be updated in H5Z_sz3_set_local automatically + conf.setDims(dimensions.begin(), dimensions.end()); + // Set compression related attributes here +// conf.cmprAlgo = SZ3::ALGO_BIOMD; + + std::vector hdims(conf.dims.begin(), conf.dims.end()); /*Create dataspace. Setting maximum size */ - if (0 > (sid = H5Screate_simple(dim, dims, NULL))) { + if (0 > (sid = H5Screate_simple(conf.N, hdims.data(), NULL))) { printf("Error in H5Screate_simple"); exit(0); } - /* setup dataset creation properties */ if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) { printf("Error in H5Pcreate"); exit(0); } - - /* Add the SZ compression filter and set the chunk size */ - if (0 > H5Pset_filter(cpid, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values)) { - printf("Error in H5Pcreate"); + + set_SZ3_conf_to_H5(cpid, conf); + + if (0 > H5Zregister(H5PLget_plugin_info())) { + printf("Error in H5Zregister"); exit(0); } - avail = H5Zfilter_avail(H5Z_FILTER_SZ3); - if (avail) { - status = H5Zget_filter_info(H5Z_FILTER_SZ3, &filter_config); - - if (filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED) + if (H5Zfilter_avail(H5Z_FILTER_SZ3)) { + unsigned filter_config; + if (H5Zget_filter_info(H5Z_FILTER_SZ3, &filter_config) & H5Z_FILTER_CONFIG_ENCODE_ENABLED) printf("sz filter is available for encoding and decoding.\n"); } - if (0 > H5Pset_chunk(cpid, dim, chunk)) { + + /* set the chunk size*/ + std::vector hchunk(hdims); +// hchunk[0] = 10; + if (0 > H5Pset_chunk(cpid, conf.N, hchunk.data())) { printf("Error in H5Pcreate"); exit(0); } - - //Initialize the configuration for SZ - //You can also use the global variable conf_params to set the configuration for sz without cfgFile. - //Example of setting an absolute error bound: - // sz_params* params = H5Z_SZ_Init_Default(); - // params->errorBoundMode = ABS; - // params->absErrBound = 1E-4; - - //H5Z_SZ_Init(cfgFile); - - printf("....Writing SZ compressed data.............\n"); - - if (dataType == SZ_FLOAT) { - float *data = new float[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%f ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_IEEE_F32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - }; - } else if (dataType == SZ_DOUBLE) { - double *data = new double[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%f ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F64LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F64BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_IEEE_F64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - }; - } else if (dataType == SZ_INT8) { - int8_t *data = new int8_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%d ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I8LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I8BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_UINT8) { - uint8_t *data = new uint8_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%d ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U8LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U8BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_INT16) { - - int16_t *data = new int16_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%d ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I16LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I16BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_UINT16) { - uint16_t *data = new uint16_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%d ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U16LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U16BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_INT32) { - //printf("%i \t %i\n", sizeof(int), sizeof(int32_t)); - int32_t *data = new int32_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%d ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I32BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_UINT32) { - uint32_t *data = new uint32_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%d ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U32BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_INT64) { - int64_t *data = new int64_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%ld ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I64LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I64BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_I64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else if (dataType == SZ_UINT64) { - uint64_t *data = new uint64_t[nbEle]; - readfile(oriFilePath, nbEle, data); - - printf("original data = "); - for (i = 0; i < 20; i++) - printf("%ld ", data[i]); - printf("....\n"); - - if (dataEndianType == LITTLE_ENDIAN_DATA) { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U64LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } else //BIG_ENDIAN_DATA - { - if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U64BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { - printf("Error in H5Dcreate"); - exit(0); - } - if (0 > H5Dwrite(idsid, H5T_STD_U64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { - printf("Error in H5Dwrite"); - exit(0); - } - } - delete[] data; - if (0 > H5Dclose(idsid)) { - printf("Error in H5Dclose"); - exit(0); - } - } else { - printf("Error: unknown data type in sz3ToHDF5.cpp!\n"); - exit(0); + + {//This is an example to get/set SZ configuration from HDF5 file + SZ3::Config conf1; + get_SZ3_conf_from_H5(cpid, conf1); +// conf1.absErrorBound = 1; + set_SZ3_conf_to_H5(cpid, conf1); } - + printf("....Writing SZ compressed data.............\n"); + + switch (dataType) { + case SZ_FLOAT:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_IEEE_F32LE, H5T_IEEE_F32BE); + break; + case SZ_DOUBLE:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_IEEE_F64LE, H5T_IEEE_F64BE); + break; + case SZ_INT8:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I8LE, H5T_STD_I8BE); + break; + case SZ_UINT8:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U8LE, H5T_STD_U8BE); + break; + case SZ_INT16:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I16LE, H5T_STD_I16BE); + break; + case SZ_UINT16:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U16LE, H5T_STD_U16BE); + break; + case SZ_INT32:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I32LE, H5T_STD_I32BE); + break; + case SZ_UINT32:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U32LE, H5T_STD_U32BE); + break; + case SZ_INT64:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I64LE, H5T_STD_I64BE); + break; + case SZ_UINT64:process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U64LE, H5T_STD_U64BE); + break; + default:std::cerr << "Error: Unknown data type\n"; + exit(EXIT_FAILURE); + } + /*Close and release resources*/ if (0 > H5Sclose(sid)) { printf("Error in H5Sclose"); @@ -542,10 +211,8 @@ int main(int argc, char *argv[]) { printf("Error in H5Fclose"); exit(0); } - free(cd_values); printf("Output hdf5 file: %s\n", outputFilePath); - herr_t ret = H5Zunregister(H5Z_FILTER_SZ3); - if (ret < 0) return -1; + if (H5Zunregister(H5Z_FILTER_SZ3) < 0) return -1; H5close(); return 0; } diff --git a/tools/mdz/include/mdz.hpp b/tools/mdz/include/mdz.hpp index a7c64892..b4a71ceb 100644 --- a/tools/mdz/include/mdz.hpp +++ b/tools/mdz/include/mdz.hpp @@ -5,7 +5,8 @@ #ifndef SZ_MDZ_H #define SZ_MDZ_H -#include +#include +#include #include #include #include @@ -17,163 +18,165 @@ #include #include #include -#include #include -#include "KmeansUtil.hpp" -#include "TimeBasedFrontend.hpp" -#include "ExaaltCompressor.hpp" +#include "SZ3/utils/KmeansUtil.hpp" +#include "SZ3/decomposition/TimeSeriesDecomposition.hpp" +#include "SZ3/compressor/specialized/SZExaaltCompressor.hpp" +using namespace SZ3; double total_compress_time = 0; double total_decompress_time = 0; const char *compressor_names[] = {"VQ", "VQT", "MT", "LR", "TS"}; -template -std::shared_ptr> -make_sz2(const SZ3::Config &conf, Predictor predictor) { - return SZ3::make_sz_general_compressor( - SZ3::SZGeneralFrontend>(conf, predictor, - SZ3::LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), - SZ3::HuffmanEncoder(), - SZ3::Lossless_zstd()); -// return new SZ3::SZGeneralCompressor>, -// SZ3::HuffmanEncoder, SZ3::Lossless_zstd>( -// SZ3::SZGeneralFrontend>(conf, predictor, -// SZ3::LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), -// SZ3::HuffmanEncoder(), -// SZ3::Lossless_zstd()); +template +std::shared_ptr> +make_sz2(const Config &conf, Predictor predictor) { + return make_compressor_sz_iterate( + conf, predictor, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), + HuffmanEncoder(), + Lossless_zstd()); +// return new SZGeneralCompressor>, +// HuffmanEncoder, Lossless_zstd>( +// SZGeneralFrontend>(conf, predictor, +// LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), +// HuffmanEncoder(), +// Lossless_zstd()); } -template -std::shared_ptr> -make_sz_timebased2(const SZ3::Config &conf, Predictor predictor, T *data_ts0) { - return SZ3::make_sz_general_compressor( - SZ3::TimeBasedFrontend>(conf, predictor, - SZ3::LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), - data_ts0), - SZ3::HuffmanEncoder(), - SZ3::Lossless_zstd()); -// return std::make_shared>, -// SZ3::HuffmanEncoder, SZ3::Lossless_zstd>( -// SZ3::TimeBasedFrontend>(conf, predictor, -// SZ3::LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), +template +std::shared_ptr> +make_sz_timebased2(const Config &conf, Predictor predictor, T *data_ts0) { + return make_compressor_sz_generic( + make_decomposition_timeseries(conf, predictor, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), + data_ts0), + HuffmanEncoder(), + Lossless_zstd()); +// return std::make_shared>, +// HuffmanEncoder, Lossless_zstd>( +// TimeSeriesDecomposition>(conf, predictor, +// LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), // data_ts0), -// SZ3::HuffmanEncoder(), -// SZ3::Lossless_zstd()); +// HuffmanEncoder(), +// Lossless_zstd()); } -template -std::shared_ptr> -make_sz_timebased(const SZ3::Config &conf, T *data_ts0) { - std::vector>> predictors; +template +std::shared_ptr> +make_sz_timebased(const Config &conf, T *data_ts0) { + std::vector>> predictors; int use_single_predictor = (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1; if (conf.lorenzo) { if (use_single_predictor) { - return make_sz_timebased2(conf, SZ3::LorenzoPredictor(conf.absErrorBound), data_ts0); + return make_sz_timebased2(conf, LorenzoPredictor(conf.absErrorBound), data_ts0); } else { - predictors.push_back(std::make_shared>(conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.absErrorBound)); } } if (conf.lorenzo2) { if (use_single_predictor) { - return make_sz_timebased2(conf, SZ3::LorenzoPredictor(conf.absErrorBound), data_ts0); + return make_sz_timebased2(conf, LorenzoPredictor(conf.absErrorBound), data_ts0); } else { - predictors.push_back(std::make_shared>(conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.absErrorBound)); } } if (conf.regression) { if (use_single_predictor) { - return make_sz_timebased2(conf, SZ3::RegressionPredictor(conf.blockSize, conf.absErrorBound), data_ts0); + return make_sz_timebased2(conf, RegressionPredictor(conf.blockSize, conf.absErrorBound), data_ts0); } else { - predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); } } - return make_sz_timebased2(conf, SZ3::ComposedPredictor(predictors), data_ts0); + return make_sz_timebased2(conf, ComposedPredictor(predictors), data_ts0); } -template -std::shared_ptr> -make_sz(const SZ3::Config &conf) { - std::vector>> predictors; +template +std::shared_ptr> +make_sz(const Config &conf) { + std::vector>> predictors; int use_single_predictor = (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1; if (conf.lorenzo) { if (use_single_predictor) { - return make_sz2(conf, SZ3::LorenzoPredictor(conf.absErrorBound)); + return make_sz2(conf, LorenzoPredictor(conf.absErrorBound)); } else { - predictors.push_back(std::make_shared>(conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.absErrorBound)); } } if (conf.lorenzo2) { if (use_single_predictor) { - return make_sz2(conf, SZ3::LorenzoPredictor(conf.absErrorBound)); + return make_sz2(conf, LorenzoPredictor(conf.absErrorBound)); } else { - predictors.push_back(std::make_shared>(conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.absErrorBound)); } } if (conf.regression) { if (use_single_predictor) { - return make_sz2(conf, SZ3::RegressionPredictor(conf.blockSize, conf.absErrorBound)); + return make_sz2(conf, RegressionPredictor(conf.blockSize, conf.absErrorBound)); } else { - predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); } } - return make_sz2(conf, SZ3::ComposedPredictor(predictors)); + return make_sz2(conf, ComposedPredictor(predictors)); } -template +template float * -VQ(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, +VQ(Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, int method, float level_start, float level_offset, int level_num) { if (level_num == 0) { printf("VQ/VQT not availble on current dataset, please use ADP or MT\n"); exit(0); } - auto sz = SZ3::SZ_Exaalt_Compressor, SZ3::HuffmanEncoder, - SZ3::Lossless_zstd>(conf, SZ3::LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), - SZ3::HuffmanEncoder(), SZ3::Lossless_zstd(), method); - sz.set_level(level_start, level_offset, level_num); + auto sz = make_compressor_exaalt(LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), + HuffmanEncoder(), Lossless_zstd(), method); + sz->set_level(level_start, level_offset, level_num); - SZ3::Timer timer(true); - SZ3::uchar *compressed; - compressed = sz.compress(data, compressed_size); + Timer timer(true); + compressed_size = conf.num * sizeof(T); + auto compressed = (uchar *) malloc(compressed_size); + sz->compress(conf, data, compressed, compressed_size); total_compress_time += timer.stop("Compression"); if (!decom) { - delete[]compressed; + free(compressed); return nullptr; } - auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; +// auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; // std::cout << "Compression Ratio = " << ratio << std::endl; // std::cout << "Compressed size = " << compressed_size << std::endl; timer.start(); - auto ts_dec_data = sz.decompress(compressed, compressed_size); + auto ts_dec_data = new T[conf.num]; + sz->decompress(conf, compressed, compressed_size, ts_dec_data); total_decompress_time += timer.stop("Decompression"); - delete[]compressed; + free(compressed); return ts_dec_data; } -template +template float * -MT(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *ts0) { +MT(Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *ts0) { // printf("eb=%.8f\n", conf.eb); auto sz = make_sz_timebased(conf, ts0); - SZ3::uchar *compressed; - SZ3::Timer timer(true); - compressed = sz->compress(conf, data, compressed_size); + Timer timer(true); + compressed_size = conf.num * sizeof(T); + auto compressed = (uchar *) malloc(compressed_size); + sz->compress(conf, data, compressed, compressed_size); total_compress_time += timer.stop("Compression"); if (!decom) { - delete[] compressed; + free(compressed); return nullptr; } @@ -183,25 +186,28 @@ MT(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T timer.start(); - auto ts_dec_data = sz->decompress(compressed, compressed_size, conf.num); + auto ts_dec_data = new T[conf.num]; + sz->decompress(conf, compressed, compressed_size, ts_dec_data); total_decompress_time += timer.stop("Decompression"); // delete sz; - delete[] compressed; + free(compressed); return ts_dec_data; } -template +template float * -SZ2(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) { +SZ2(Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) { auto sz = make_sz(conf); - SZ3::uchar *compressed; - SZ3::Timer timer(true); - compressed = sz->compress(conf, data, compressed_size); + Timer timer(true); + compressed_size = conf.num * sizeof(T); + auto compressed = (uchar *) malloc(compressed_size); + sz->compress(conf, data, compressed, compressed_size); + total_compress_time += timer.stop("Compression"); if (!decom) { - delete[] compressed; + free(compressed); return nullptr; } @@ -210,19 +216,20 @@ SZ2(SZ3::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) { // std::cout << "Compressed size = " << compressed_size << std::endl; timer.start(); - auto ts_dec_data = sz->decompress(compressed, compressed_size, conf.num); + auto ts_dec_data = new T[conf.num]; + sz->decompress(conf, compressed, compressed_size, ts_dec_data); total_decompress_time += timer.stop("Decompression"); // delete sz; - delete[] compressed; + free(compressed); return ts_dec_data; } -template -void select(SZ3::Config conf, int &method, size_t ts, T *data_all, +template +void select(Config conf, int &method, size_t ts, T *data_all, float level_start, float level_offset, int level_num, T *data_ts0, size_t batch_size) { -// && (ts_last_select == -1 || t - ts_last_select >= conf.batch_size * 10)) { +// && (ts_last_select == -1 || t - ts_last_select >= conf.timestep_batch * 10)) { // std::cout << "****************** BEGIN Selection ****************" << std::endl; // ts_last_select = ts; std::vector compressed_size(10, std::numeric_limits::max()); @@ -287,35 +294,36 @@ std::unique_ptr readfile(const char *file, size_t start, size_t num) { } -template -SZ3::uchar *LAMMPS_compress(SZ3::Config conf, T *data, int method, size_t &compressed_size, - float level_start, float level_offset, int level_num, T *ts0) { +template +uchar *LAMMPS_compress(Config conf, T *data, int method, size_t &compressed_size, + float level_start, float level_offset, int level_num, T *ts0) { if ((method == 0 || method == 1) && level_num == 0) { printf("VQ/VQT not available on current dataset, please use ADP or MT\n"); exit(0); } - SZ3::uchar *compressed_data; + compressed_size = conf.num * sizeof(T); + auto compressed_data = new uchar[compressed_size]; + std::shared_ptr> sz; if (method == 0 || method == 1) { - auto sz = SZ3::SZ_Exaalt_Compressor, SZ3::HuffmanEncoder, SZ3::Lossless_zstd>( - conf, SZ3::LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), - SZ3::HuffmanEncoder(), SZ3::Lossless_zstd(), method); - sz.set_level(level_start, level_offset, level_num); - compressed_data = sz.compress(data, compressed_size); + sz = make_compressor_exaalt(LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), + HuffmanEncoder(), Lossless_zstd(), method); + sz->set_level(level_start, level_offset, level_num); } else if (method == 2 || method == 4) { - auto sz = make_sz_timebased(conf, ts0); - compressed_data = sz->compress(conf, data, compressed_size); + sz = make_sz_timebased(conf, ts0); +// sz->compress(conf, data, compressed_size); } else { - auto sz = make_sz(conf); - compressed_data = sz->compress(conf, data, compressed_size); + sz = make_sz(conf); +// sz->compress(conf, data, compressed_size); } + sz->compress(conf, data, compressed_data, compressed_size); // auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; // std::cout << "Compression Ratio = " << ratio << std::endl; // std::cout << "Compressed size = " << compressed_size << std::endl; return compressed_data; } -template -int LAMMPS_select_compressor(SZ3::Config conf, T *data, bool firsttime, +template +int LAMMPS_select_compressor(Config conf, T *data, bool firsttime, float level_start, float level_offset, int level_num, T *data_ts0) { // std::cout << "****************** BEGIN Selection ****************" << std::endl; @@ -332,7 +340,7 @@ int LAMMPS_select_compressor(SZ3::Config conf, T *data, bool firsttime, } std::vector data1; - SZ3::uchar *cmpr; + uchar *cmpr; if (level_num > 0) { data1 = std::vector(data, data + conf.num); @@ -363,9 +371,9 @@ int LAMMPS_select_compressor(SZ3::Config conf, T *data, bool firsttime, } -template +template inline typename std::enable_if::type -MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) { +MDZ_Compress(Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) { // if (N != 2) { // throw std::invalid_argument("dimension should be 2"); // } @@ -397,8 +405,8 @@ MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, in size_t sample_num = 0.1 * conf.dims[1]; sample_num = std::min(sample_num, (size_t) 20000); sample_num = std::max(sample_num, std::min((size_t) 5000, conf.dims[1])); - SZ3::get_cluster(input_data, conf.dims[1], level_start, level_offset, level_num, - sample_num); + get_cluster(input_data, conf.dims[1], level_start, level_offset, level_num, + sample_num); if (level_num > conf.dims[1] * 0.25) { level_num = 0; } @@ -420,9 +428,9 @@ MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, in T max = *std::max_element(data, data + conf.num); T min = *std::min_element(data, data + conf.num); - if (conf.errorBoundMode == SZ3::EB_ABS) { + if (conf.errorBoundMode == EB_ABS) { conf.relErrorBound = conf.absErrorBound / (max - min); - } else if (conf.errorBoundMode == SZ3::EB_REL) { + } else if (conf.errorBoundMode == EB_REL) { conf.absErrorBound = conf.relErrorBound * (max - min); } @@ -458,18 +466,19 @@ MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, in } if (lossless_first_frame) { auto zstd = SZ3::Lossless_zstd(); - size_t outSize = conf.dims[1] * sizeof(T); - auto cmpr = zstd.compress((SZ3::uchar *) data_ts0.data(), outSize, outSize); - delete[] cmpr; -// printf("outsize %lu\n", outSize); - total_compressed_size += outSize; + size_t inSize = conf.dims[1] * sizeof(T); + uchar *buffer = new uchar[inSize]; + auto cmpSize = zstd.compress((SZ3::uchar *) data_ts0.data(), inSize, buffer, inSize); + delete[] buffer; +// printf("outsize %lu\n", cmpSize); + total_compressed_size += cmpSize; } return total_compressed_size; } -template -inline typename std::enable_if::type MDZ_Compress(SZ3::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) { +template +inline typename std::enable_if::type MDZ_Compress(Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) { size_t total_compressed_size = 0; auto dims = conf.dims; std::vector input(conf.num), output(conf.num); @@ -481,7 +490,7 @@ inline typename std::enable_if::type MDZ_Compress(SZ3::Config co } } for (int i = 0; i < conf.dims[2]; i++) { - SZ3::Config conf_2D(conf); + Config conf_2D(conf); conf_2D.dims = {conf.dims[0], conf.dims[1]}; conf_2D.num = conf.dims[0] * conf.dims[1]; total_compressed_size += MDZ_Compress(conf_2D, input.data() + i * conf.dims[0] * conf.dims[1], diff --git a/tools/mdz/mdz.cpp b/tools/mdz/mdz.cpp index 2f311d9b..b7ac4249 100644 --- a/tools/mdz/mdz.cpp +++ b/tools/mdz/mdz.cpp @@ -2,7 +2,6 @@ #include - void usage() { printf("Usage: \n"); printf("For 1D input: mdz file_path -1 n_atoms -r reb\n"); diff --git a/tools/sz3/CMakeLists.txt b/tools/sz3/CMakeLists.txt index ff50223c..59050ec9 100644 --- a/tools/sz3/CMakeLists.txt +++ b/tools/sz3/CMakeLists.txt @@ -8,5 +8,6 @@ foreach (SRCFILE IN LISTS cpu_source_files) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${EXE}$ TYPE BIN) endforeach () - install(FILES testfloat_8_8_128.dat DESTINATION ${CMAKE_INSTALL_DATADIR}/SZ3) + +#add_subdirectory(demo) diff --git a/tools/sz3/demo/CMakeLists.txt b/tools/sz3/demo/CMakeLists.txt new file mode 100644 index 00000000..c7831de4 --- /dev/null +++ b/tools/sz3/demo/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB cpu_source_files "${CMAKE_SOURCE_DIR}/tools/sz3/demo/*.cpp") + +foreach (SRCFILE IN LISTS cpu_source_files) + get_filename_component(EXE ${SRCFILE} NAME_WLE) + message(STATUS "Target = ${EXE}") + add_executable(${EXE} ${EXE}.cpp) + target_link_libraries(${EXE} PRIVATE SZ3) +# install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${EXE}$ TYPE BIN) + +endforeach () \ No newline at end of file diff --git a/tools/sz3/demo/sz3demo_customized.cpp b/tools/sz3/demo/sz3demo_customized.cpp new file mode 100644 index 00000000..716658be --- /dev/null +++ b/tools/sz3/demo/sz3demo_customized.cpp @@ -0,0 +1,109 @@ +#include "SZ3/api/sz.hpp" + +using namespace SZ3; + +template +class MyDecomposition : public concepts::DecompositionInterface { +public: + MyDecomposition(const Config &conf) : num(conf.num) {} + + std::vector compress(const Config &conf, T *data) { + //write your own logic here + std::vector output(num); + for (size_t i = 0; i < num; i++) { + output[i] = data[i]; + } + return output; + } + + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) { + //write your own logic here + for (size_t i = 0; i < num; i++) { + dec_data[i] = quant_inds[i]; + } + return dec_data; + } + + void save(uchar *&c) { + write(num, c); + } + + void load(const uchar *&c, size_t &remaining_length) { + read(num, c, remaining_length); + } + +private: + size_t num; +}; + +template +void SZ3_customized_compress(Config &conf, T *data, char *dst, size_t &outSize) { + + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_generic( + MyDecomposition(conf), + HuffmanEncoder(), + Lossless_zstd()); + + sz->compress(conf, data, (uchar *) dst, outSize); +} + +template +void SZ3_customized_decompress(const Config &conf, const char *cmpData, size_t cmpSize, T *decData) { + uchar const *cmpDataPos = (uchar *) cmpData; + auto sz = make_compressor_sz_generic( + MyDecomposition(conf), + HuffmanEncoder(), + Lossless_zstd()); + + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + + +int main(int argc, char **argv) { + + + std::vector dims({100, 200, 300}); + Config conf({dims[0], dims[1], dims[2]}); + conf.cmprAlgo = ALGO_INTERP_LORENZO; + conf.errorBoundMode = EB_ABS; // refer to def.hpp for all supported error bound mode + conf.absErrorBound = 1E-3; // absolute error bound 1e-3 + + + std::vector input_data(conf.num); + std::vector dec_data(conf.num); + std::vector stride({dims[1] * dims[2], dims[2], 1}); + + for (size_t i = 0; i < dims[0]; ++i) { + for (size_t j = 0; j < dims[1]; ++j) { + for (size_t k = 0; k < dims[2]; ++k) { + double x = static_cast(i) - static_cast(dims[0]) / 2.0; + double y = static_cast(j) - static_cast(dims[1]) / 2.0; + double z = static_cast(k) - static_cast(dims[2]) / 2.0; + input_data[i * stride[0] + j * stride[1] + k] = static_cast(.0001 * y * sin(y) + .0005 * cos(pow(x, 2) + x) + z); + } + } + } + + std::vector input_data_copy(input_data); + + auto dec_data_p = dec_data.data(); + std::vector cmpData(conf.num); + size_t cmpSize = cmpData.size(); + + SZ3_customized_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ3_customized_decompress(conf, cmpData.data(), cmpSize, dec_data_p); + +// double max_err = 0.0; +// for (size_t i = 0; i < conf.num; i++) { +// if (fabs(dec_data[i] - input_data_copy[i]) > max_err) { +// max_err = fabs(dec_data[i] - input_data_copy[i]); +// } +// } + printf("Smoke test %s", true ? "passed" : "failed"); +// printf("%lu ", conf.num); + return 0; + + +} diff --git a/tools/sz3/demo/sz3demo_interpolation.cpp b/tools/sz3/demo/sz3demo_interpolation.cpp new file mode 100644 index 00000000..401261c7 --- /dev/null +++ b/tools/sz3/demo/sz3demo_interpolation.cpp @@ -0,0 +1,75 @@ +#include "SZ3/api/sz.hpp" + +using namespace SZ3; + +template +void SZ3_interpolation_compress(Config &conf, T *data, char *dst, size_t &outSize) { + + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + sz->compress(conf, data, (uchar *) dst, outSize); +} + +template +void SZ3_interpolation_decompress(const Config &conf, const char *cmpData, size_t cmpSize, T *decData) { + uchar const *cmpDataPos = (uchar *) cmpData; + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + + +int main(int argc, char **argv) { + + + std::vector dims({100, 200, 300}); + Config conf({dims[0], dims[1], dims[2]}); + conf.cmprAlgo = ALGO_INTERP_LORENZO; + conf.errorBoundMode = EB_ABS; // refer to def.hpp for all supported error bound mode + conf.absErrorBound = 1E-3; // absolute error bound 1e-3 + + + std::vector input_data(conf.num); + std::vector dec_data(conf.num); + std::vector stride({dims[1] * dims[2], dims[2], 1}); + + for (size_t i = 0; i < dims[0]; ++i) { + for (size_t j = 0; j < dims[1]; ++j) { + for (size_t k = 0; k < dims[2]; ++k) { + double x = static_cast(i) - static_cast(dims[0]) / 2.0; + double y = static_cast(j) - static_cast(dims[1]) / 2.0; + double z = static_cast(k) - static_cast(dims[2]) / 2.0; + input_data[i * stride[0] + j * stride[1] + k] = static_cast(.0001 * y * sin(y) + .0005 * cos(pow(x, 2) + x) + z); + } + } + } + + std::vector input_data_copy(input_data); + + std::vector cmpData(conf.num); + size_t cmpSize = cmpData.size(); + auto dec_data_p = dec_data.data(); + + SZ3_interpolation_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ3_interpolation_decompress(conf, cmpData.data(), cmpSize, dec_data_p); + + double max_err = 0.0; + for (size_t i = 0; i < conf.num; i++) { + if (fabs(dec_data[i] - input_data_copy[i]) > max_err) { + max_err = fabs(dec_data[i] - input_data_copy[i]); + } + } + printf("Smoke test %s", max_err <= conf.absErrorBound ? "passed" : "failed"); +// printf("%lu ", conf.num); + return 0; + + +} diff --git a/tools/sz3/demo/sz3demo_lorenzo_fast.cpp b/tools/sz3/demo/sz3demo_lorenzo_fast.cpp new file mode 100644 index 00000000..3817c1d9 --- /dev/null +++ b/tools/sz3/demo/sz3demo_lorenzo_fast.cpp @@ -0,0 +1,84 @@ +#include "SZ3/api/sz.hpp" + +/** + * Fast Lorenzo + * Only support 1D and 3D + */ +using namespace SZ3; + +template +void SZ3_lorenzo_v1_compress(Config &conf, T *data, char *dst, size_t &outSize) { + + calAbsErrorBound(conf, data); + if (N != 1 && N != 3) { + throw std::invalid_argument("Lorenzo v1 only support 1D and 3D data input"); + } + conf.lorenzo = true; + conf.lorenzo2 = false; + conf.regression = false; + conf.regression2 = false; + auto sz = make_compressor_sz_generic( + make_decomposition_lorenzo_regression(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + + sz->compress(conf, data, (uchar *) dst, outSize); +} + +template +void SZ3_lorenzo_v1_decompress(const Config &conf, const char *cmpData, size_t cmpSize, T *decData) { + uchar const *cmpDataPos = (uchar *) cmpData; + auto sz = make_compressor_sz_generic( + make_decomposition_lorenzo_regression(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), + Lossless_zstd()); + + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + +int main(int argc, char **argv) { + + + std::vector dims({100, 200, 300}); + Config conf({dims[0], dims[1], dims[2]}); + conf.cmprAlgo = ALGO_INTERP_LORENZO; + conf.errorBoundMode = EB_ABS; // refer to def.hpp for all supported error bound mode + conf.absErrorBound = 1E-3; // absolute error bound 1e-3 + + + std::vector input_data(conf.num); + std::vector dec_data(conf.num); + std::vector stride({dims[1] * dims[2], dims[2], 1}); + + for (size_t i = 0; i < dims[0]; ++i) { + for (size_t j = 0; j < dims[1]; ++j) { + for (size_t k = 0; k < dims[2]; ++k) { + double x = static_cast(i) - static_cast(dims[0]) / 2.0; + double y = static_cast(j) - static_cast(dims[1]) / 2.0; + double z = static_cast(k) - static_cast(dims[2]) / 2.0; + input_data[i * stride[0] + j * stride[1] + k] = static_cast(.0001 * y * sin(y) + .0005 * cos(pow(x, 2) + x) + z); + } + } + } + + std::vector input_data_copy(input_data); + + auto dec_data_p = dec_data.data(); + + std::vector cmpData(conf.num); + size_t cmpSize = cmpData.size(); + SZ3_lorenzo_v1_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ3_lorenzo_v1_decompress(conf, cmpData.data(), cmpSize, dec_data_p); + + double max_err = 0.0; + for (size_t i = 0; i < conf.num; i++) { + if (fabs(dec_data[i] - input_data_copy[i]) > max_err) { + max_err = fabs(dec_data[i] - input_data_copy[i]); + } + } + printf("Smoke test %s", max_err <= conf.absErrorBound ? "passed" : "failed"); +// printf("%lu ", conf.num); + return 0; + + +} diff --git a/tools/sz3/demo/sz3demo_lorenzo_generic.cpp b/tools/sz3/demo/sz3demo_lorenzo_generic.cpp new file mode 100644 index 00000000..f496fff2 --- /dev/null +++ b/tools/sz3/demo/sz3demo_lorenzo_generic.cpp @@ -0,0 +1,81 @@ +#include "SZ3/api/sz.hpp" +/** + * Slow Lorenzo + * support any data dimension + */ +using namespace SZ3; + +template +void SZ3_lorenzo_v2_compress(Config &conf, T *data, char *dst, size_t &outSize) { + + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_iterate( //compressor + conf, + LorenzoPredictor(conf.absErrorBound), //predictor + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), //quantizer + HuffmanEncoder(),//encoder + Lossless_zstd()); //lossless + sz->compress(conf, data, (uchar *) dst, outSize); +} + +template +void SZ3_lorenzo_v2_decompress(const Config &conf, const char *cmpData, size_t cmpSize, T *decData) { + auto cmpDataPos = (uchar *) cmpData; + auto sz = make_compressor_sz_iterate( //compressor + conf, + LorenzoPredictor(conf.absErrorBound), //predictor + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), //quantizer + HuffmanEncoder(),//encoder + Lossless_zstd()); //lossless + + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + +int main(int argc, char **argv) { + + + std::vector dims({100, 200, 300}); + Config conf({dims[0], dims[1], dims[2]}); + conf.cmprAlgo = ALGO_INTERP_LORENZO; + conf.errorBoundMode = EB_ABS; // refer to def.hpp for all supported error bound mode + conf.absErrorBound = 1E-3; // absolute error bound 1e-3 + + + std::vector input_data(conf.num); + std::vector dec_data(conf.num); + std::vector stride({dims[1] * dims[2], dims[2], 1}); + + for (size_t i = 0; i < dims[0]; ++i) { + for (size_t j = 0; j < dims[1]; ++j) { + for (size_t k = 0; k < dims[2]; ++k) { + double x = static_cast(i) - static_cast(dims[0]) / 2.0; + double y = static_cast(j) - static_cast(dims[1]) / 2.0; + double z = static_cast(k) - static_cast(dims[2]) / 2.0; + input_data[i * stride[0] + j * stride[1] + k] = static_cast(.0001 * y * sin(y) + .0005 * cos(pow(x, 2) + x) + z); + } + } + } + + std::vector input_data_copy(input_data); + + + auto dec_data_p = dec_data.data(); + + std::vector cmpData(conf.num); + size_t cmpSize = cmpData.size(); + SZ3_lorenzo_v2_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ3_lorenzo_v2_decompress(conf, cmpData.data(), cmpSize, dec_data_p); + + double max_err = 0.0; + for (size_t i = 0; i < conf.num; i++) { + if (fabs(dec_data[i] - input_data_copy[i]) > max_err) { + max_err = fabs(dec_data[i] - input_data_copy[i]); + } + } + printf("Smoke test %s", max_err <= conf.absErrorBound ? "passed" : "failed"); +// printf("%lu ", conf.num); + return 0; + + +} diff --git a/tools/sz3/deprecated/beta_sz_pw.cpp b/tools/sz3/deprecated/beta_sz_pw.cpp index 3ed8d977..8088bbbd 100644 --- a/tools/sz3/deprecated/beta_sz_pw.cpp +++ b/tools/sz3/deprecated/beta_sz_pw.cpp @@ -7,7 +7,7 @@ #include "SZ3/utils/FileUtil.hpp" #include "SZ3/utils/Config.hpp" #include "SZ3/def.hpp" -#include "SZ3/compressor/SZGeneralCompressor.hpp" +#include "SZ3/compressor/SZIterateCompressor.hpp" #include "SZ3/encoder/HuffmanEncoder.hpp" #include "SZ3/encoder/ArithmeticEncoder.hpp" #include "SZ3/encoder/BypassEncoder.hpp" @@ -89,7 +89,7 @@ float SZ_compress(std::unique_ptr const &data, } // std::vector data1 = std::vector(data.get(), data.get() + conf.num); - auto sz = SZ3::make_sz_general_compressor(frontend, encoder, lossless); + auto sz = SZ3::make_compressor_sz_iterate(frontend, encoder, lossless); SZ3::Timer timer; timer.start(); diff --git a/tools/sz3/deprecated/beta_sz_truncate.cpp b/tools/sz3/deprecated/beta_sz_truncate.cpp index 994a6509..90a90396 100644 --- a/tools/sz3/deprecated/beta_sz_truncate.cpp +++ b/tools/sz3/deprecated/beta_sz_truncate.cpp @@ -1,4 +1,4 @@ -#include "SZ3/compressor/SZTruncateCompressor.hpp" +#include "SZ3/compressor/specialized/SZTruncateCompressor.hpp" #include "SZ3/lossless/Lossless_zstd.hpp" #include "SZ3/lossless/Lossless_bypass.hpp" #include "SZ3/utils/FileUtil.hpp" diff --git a/tools/sz3/sz3_smoke_test.cpp b/tools/sz3/sz3_smoke_test.cpp index 8848dc0e..6c06d155 100644 --- a/tools/sz3/sz3_smoke_test.cpp +++ b/tools/sz3/sz3_smoke_test.cpp @@ -31,7 +31,7 @@ int main(int argc, char **argv) { } std::vector input_data_copy(input_data); - + conf.openmp = true; size_t cmpSize; char *cmpData = SZ_compress(conf, input_data.data(), cmpSize); auto dec_data_p = dec_data.data();