Merge branch 'refs/heads/master' into bio

# Conflicts: # include/SZ3/api/impl/SZDispatcher.hpp # include/SZ3/compressor/SZGeneralCompressor.hpp # include/SZ3/decomposition/SZBioMDFrontend.hpp # include/SZ3/decomposition/SZBioMDXtcBasedFrontend.hpp # include/SZ3/encoder/HuffmanEncoder.hpp # include/SZ3/lossless/Lossless_zstd.hpp # include/SZ3/utils/ByteUtil.hpp # include/SZ3/utils/Config.hpp # tools/H5Z-SZ3/src/H5Z_SZ3.cpp # tools/H5Z-SZ3/test/dsz3FromHDF5.cpp # tools/H5Z-SZ3/test/sz3ToHDF5.cpp
szcompressor · Aug 27, 2024 · dc2b5ea · dc2b5ea
2 parents d2fb5e5 + b3dab40
commit dc2b5ea
Show file tree

Hide file tree

Showing 60 changed files with 2,291 additions and 1,726 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,11 @@
 cmake_minimum_required(VERSION 3.18)
-project(SZ3 VERSION 3.1.8)
+project(SZ3 VERSION 3.2.0)
+
+#data version defines the version of the compressed data format
+#it is not always equal to the program version (e.g., SZ3 v3.1.0 and SZ3 v.3.1.1 may use the same data version of v.3.1.0)
+#only update data version if the new version of the program changes compressed data format
+set(SZ3_DATA_VERSION 3.2.0)
+
 include(GNUInstallDirs)
 include(CTest)
 
@@ -19,8 +25,9 @@ endif ()
 
 find_package(PkgConfig)
 
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp)
+configure_file(
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in
+        ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp)
 
 add_library(${PROJECT_NAME} INTERFACE)
 target_include_directories(
@@ -31,7 +38,7 @@ target_include_directories(
 )
 target_compile_features(${PROJECT_NAME}
         INTERFACE cxx_std_17
-        )
+)
 
 find_package(OpenMP)
 if (OpenMP_FOUND)
@@ -84,18 +91,18 @@ install(TARGETS ${PROJECT_NAME}
         EXPORT SZ3Targets
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
         INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-        )
+)
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES
         "${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp"
         DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SZ3/"
-        )
+)
 install(EXPORT SZ3Targets NAMESPACE SZ3:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3)
 include(CMakePackageConfigHelpers)
 configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/SZ3Config.cmake.in
         "${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake"
         INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3
-        )
+)
 write_basic_package_version_file(
         "${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake"
         VERSION ${PROJECT_VERSION}
@@ -105,6 +112,6 @@ install(FILES
         ${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake
         ${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake
         DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3
-        )
+)
 #export sz3 target for external use
 export(TARGETS SZ3 FILE SZ3.cmake)
diff --git a/README.md b/README.md
@@ -6,45 +6,6 @@ SZ3: A Modular Error-bounded Lossy Compression Framework for Scientific Datasets
 * Supervisor: Franck Cappello
 * Other Contributors: Robert Underwood, Sihuan Li, Ali M. Gok
 
-## Citations
-
-**Kindly note**: If you mention SZ in your paper, the most appropriate citation is including these three references (**TBD22, ICDE21, Bigdata18**), because they cover the design and implementation of the latest version of SZ.
-
-* SZ3 Framework: Xin Liang, Kai Zhao, Sheng Di, Sihuan Li, Robert Underwood, Ali M Gok, Jiannan Tian, Junjing Deng, Jon C Calhoun, Dingwen Tao, Zizhong Chen, and Franck Cappello.
-  "[SZ3: A modular framework for composing prediction-based error-bounded lossy compressors](https://ieeexplore.ieee.org/abstract/document/9866018)",
-  IEEE Transactions on Big Data (TBD 22).
-
-* SZ3 Algorithm: Kai Zhao, Sheng Di, Maxim Dmitriev, Thierry-Laurent D. Tonellot, Zizhong Chen, and Franck
-  Cappello. "[Optimizing Error-Bounded Lossy Compression for Scientiﬁc Data by Dynamic Spline Interpolation](https://ieeexplore.ieee.org/document/9458791)"
-  , Proceeding of the 37th IEEE International Conference on Data Engineering (ICDE 21), Chania, Crete, Greece, Apr 19 -
-  22, 2021.
-
-* SZauto: Kai Zhao, Sheng Di, Xin Liang, Sihuan Li, Dingwen Tao, Zizhong Chen, and Franck
-  Cappello. "[Significantly Improving Lossy Compression for HPC Datasets with Second-Order Prediction and Parameter Optimization](https://dl.acm.org/doi/10.1145/3369583.3392688)"
-  , Proceedings of the 29th International Symposium on High-Performance Parallel and Distributed Computing (HPDC 20),
-  Stockholm, Sweden, 2020. (code: https://github.com/szcompressor/SZauto/)
-
-* SZ 2.0+: Xin Liang, Sheng Di, Dingwen Tao, Zizhong Chen, Franck
-  Cappello, "[Error-Controlled Lossy Compression Optimized for High Compression Ratios of Scientific Datasets](https://ieeexplore.ieee.org/document/8622520)"
-  , in IEEE International Conference on Big Data (Bigdata 2018), Seattle, WA, USA, 2018.
-
-* SZ 1.4.0-1.4.13: Dingwen Tao, Sheng Di, Franck
-  Cappello. "[Significantly Improving Lossy Compression for Scientific Data Sets Based on Multidimensional Prediction and Error-Controlled Quantization](https://ieeexplore.ieee.org/document/7967203)"
-  , in IEEE International Parallel and Distributed Processing Symposium (IPDPS 2017), Orlando, Florida, USA, 2017.
-
-* SZ 0.1-1.0: Sheng Di, Franck
-  Cappello. "[Fast Error-bounded Lossy HPC Data Compression with SZ](https://ieeexplore.ieee.org/document/7516069)", in
-  IEEE International Parallel and Distributed Processing Symposium (IPDPS 2016), Chicago, IL, USA, 2016.
-
-* Point-wise relative error bound mode (i.e., PW_REL): Xin Liang, Sheng Di, Dingwen Tao, Zizhong Chen, Franck
-  Cappello, "[An Efficient Transformation Scheme for Lossy Data Compression with Point-wise Relative Error Bound](https://ieeexplore.ieee.org/document/8514879)"
-  , in IEEE International Conference on Clustering Computing (CLUSTER 2018), Belfast, UK, 2018. (Best Paper)
-
-## 3rd party libraries/tools
-
-* Zstandard (https://facebook.github.io/zstd/). Zstandard v1.4.5 is included and will be used if libzstd can not be found by
-  pkg-config.
-
 ## Installation
 
 * mkdir build && cd build
@@ -54,6 +15,11 @@ SZ3: A Modular Error-bounded Lossy Compression Framework for Scientific Datasets
 
 Then, you'll find all the executables in [INSTALL_DIR]/bin and header files in [INSTALL_DIR]/include
 
+## 3rd party libraries/tools
+
+* Zstandard (https://facebook.github.io/zstd/). Zstandard v1.4.5 is included and will be used if libzstd can not be found by
+  pkg-config.
+
 ## Testing Examples
 
 You can use the executable 'sz3' command to do the compression/decompression.
@@ -91,26 +57,27 @@ Scripts without parameters below should work fine by replacing SZ2 with SZ3.
 * Compatible with both SZ3 and SZ2
 * Requiring SZ2/3 dynamic library
 
-## H5Z-SZ3
-
-Use examples/print_h5repack_args.c to construct the cd_values parameters based on the specified error configuration. 
-
-* Example: 
-
-Compression: 
-
-[sdi@localhost build]$ h5repack -f UD=32024,0,5,0,981668463,0,0,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5
+#### Fortran API
+* Special thanks to [Oscar Mojica](https://github.com/ofmla) for providing the Fortran API
+* Visit [this Github repository](https://github.com/ofmla/sz3_simple_example) for details
 
-Decompression:
+#### H5Z-SZ3
+* Located in 'tools/H5Z-SZ3'
+* Please add "-DBUILD_H5Z_FILTER=ON" to enable this function for CMake.
+* sz3ToHDF5 and HDF5ToSz3 are provided for testing.
 
-[sdi@localhost build]$ h5repack -f NONE -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.out.h5
+[//]: # (* Use examples/print_h5repack_args.c to construct the cd_values parameters based on the specified error configuration.)
+[//]: # ()
+[//]: # (* Compression example: )
+[//]: # (`h5repack -f UD=32024,0,5,0,981668463,0,0,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5`)
+[//]: # ()
+[//]: # (* Decompression example:)
+[//]: # (`h5repack -f NONE -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.out.h5`)
+[//]: # ()
+[//]: # (* Alternatively, the error bound information can also be given through sz3.config &#40;when there are no cd_values for h5repack&#41;. Example &#40;You need to put sz3.config in the current local directory so that it will read sz3.config to get error bounds&#41;:)
+[//]: # (`h5repack -f UD=32024,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5`)
 
 
-Alternatively, the error bound information can also be given through sz3.config (when there are no cd_values for h5repack)
-
-* Example (You need to put sz3.config in the current local directory so that it will read sz3.config to get error bounds):
-
-[sdi@localhost build]$ h5repack -f UD=32024,0 -i ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.h5 -o ~/Data/CESM-ATM-tylor/1800x3600/CLDLOW_1_1800_3600.dat.sz3.h5
 
 ## Version history
 
@@ -126,4 +93,39 @@ Version New features
 * SZ 3.1.5 Support HDF5 by H5Z-SZ3. Please add "-DBUILD_H5Z_FILTER=ON" to enable this function for CMake.
 * SZ 3.1.6 Support C API and Python API.
 * SZ 3.1.7 Initial MDZ(https://github.com/szcompressor/SZ3/tree/master/tools/mdz) support.
-* SZ 3.1.8 namespace changed from SZ to SZ3. H5Z-SZ3 supports configuration file now.
+* SZ 3.1.8 namespace changed from SZ to SZ3. H5Z-SZ3 supports configuration file now.
+* SZ 3.2.0 API reconstructed for FZ. H5Z-SZ3 rewrite. Compression version checking.
+
+## Citations
+
+**Kindly note**: If you mention SZ in your paper, the most appropriate citation is including these three references (**TBD22, ICDE21, Bigdata18**), because they cover the design and implementation of the latest version of SZ.
+
+* SZ3 Framework: Xin Liang, Kai Zhao, Sheng Di, Sihuan Li, Robert Underwood, Ali M Gok, Jiannan Tian, Junjing Deng, Jon C Calhoun, Dingwen Tao, Zizhong Chen, and Franck Cappello.
+  "[SZ3: A modular framework for composing prediction-based error-bounded lossy compressors](https://ieeexplore.ieee.org/abstract/document/9866018)",
+  IEEE Transactions on Big Data (TBD 22).
+
+* SZ3 Algorithm: Kai Zhao, Sheng Di, Maxim Dmitriev, Thierry-Laurent D. Tonellot, Zizhong Chen, and Franck
+  Cappello. "[Optimizing Error-Bounded Lossy Compression for Scientiﬁc Data by Dynamic Spline Interpolation](https://ieeexplore.ieee.org/document/9458791)"
+  , Proceeding of the 37th IEEE International Conference on Data Engineering (ICDE 21), Chania, Crete, Greece, Apr 19 -
+  22, 2021.
+
+* SZauto: Kai Zhao, Sheng Di, Xin Liang, Sihuan Li, Dingwen Tao, Zizhong Chen, and Franck
+  Cappello. "[Significantly Improving Lossy Compression for HPC Datasets with Second-Order Prediction and Parameter Optimization](https://dl.acm.org/doi/10.1145/3369583.3392688)"
+  , Proceedings of the 29th International Symposium on High-Performance Parallel and Distributed Computing (HPDC 20),
+  Stockholm, Sweden, 2020. (code: https://github.com/szcompressor/SZauto/)
+
+* SZ 2.0+: Xin Liang, Sheng Di, Dingwen Tao, Zizhong Chen, Franck
+  Cappello, "[Error-Controlled Lossy Compression Optimized for High Compression Ratios of Scientific Datasets](https://ieeexplore.ieee.org/document/8622520)"
+  , in IEEE International Conference on Big Data (Bigdata 2018), Seattle, WA, USA, 2018.
+
+* SZ 1.4.0-1.4.13: Dingwen Tao, Sheng Di, Franck
+  Cappello. "[Significantly Improving Lossy Compression for Scientific Data Sets Based on Multidimensional Prediction and Error-Controlled Quantization](https://ieeexplore.ieee.org/document/7967203)"
+  , in IEEE International Parallel and Distributed Processing Symposium (IPDPS 2017), Orlando, Florida, USA, 2017.
+
+* SZ 0.1-1.0: Sheng Di, Franck
+  Cappello. "[Fast Error-bounded Lossy HPC Data Compression with SZ](https://ieeexplore.ieee.org/document/7516069)", in
+  IEEE International Parallel and Distributed Processing Symposium (IPDPS 2016), Chicago, IL, USA, 2016.
+
+* Point-wise relative error bound mode (i.e., PW_REL): Xin Liang, Sheng Di, Dingwen Tao, Zizhong Chen, Franck
+  Cappello, "[An Efficient Transformation Scheme for Lossy Data Compression with Point-wise Relative Error Bound](https://ieeexplore.ieee.org/document/8514879)"
+  , in IEEE International Conference on Clustering Computing (CLUSTER 2018), Belfast, UK, 2018. (Best Paper)
diff --git a/include/SZ3/api/impl/SZAlgo.hpp b/include/SZ3/api/impl/SZAlgo.hpp
@@ -0,0 +1,42 @@
+#ifndef SZ3_SZALGO_HPP
+#define SZ3_SZALGO_HPP
+
+#include "SZ3/compressor/SZGenericCompressor.hpp"
+#include "SZ3/decomposition/NoPredictionDecomposition.hpp"
+#include "SZ3/quantizer/IntegerQuantizer.hpp"
+#include "SZ3/lossless/Lossless_zstd.hpp"
+#include "SZ3/encoder/HuffmanEncoder.hpp"
+#include "SZ3/utils/Config.hpp"
+
+namespace SZ3 {
+    template<class T, uint N>
+    size_t SZ_compress_nopred(Config &conf, T *data, uchar *cmpData, size_t cmpCap) {
+        assert(N == conf.N);
+        assert(conf.cmprAlgo == ALGO_INTERP);
+        calAbsErrorBound(conf, data);
+
+        auto sz = make_compressor_sz_generic<T, N>(
+                make_decomposition_noprediction<T, N>(conf,
+                                                      LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
+                HuffmanEncoder<int>(),
+                Lossless_zstd());
+        return sz->compress(conf, data, cmpData, cmpCap);
+//        return cmpData;
+    }
+
+
+    template<class T, uint N>
+    void SZ_decompress_nopred(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) {
+        assert(conf.cmprAlgo == ALGO_INTERP);
+        auto cmpDataPos = cmpData;
+        auto sz = make_compressor_sz_generic<T, N>(
+                make_decomposition_noprediction<T, N>(conf,
+                                                      LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
+                HuffmanEncoder<int>(),
+                Lossless_zstd());
+        sz->decompress(conf, cmpDataPos, cmpSize, decData);
+    }
+
+
+}
+#endif