From a2dc7e0ff3e5f2c601e1c6f685873bd8c5c84df3 Mon Sep 17 00:00:00 2001 From: Arthur Glowacki Date: Mon, 2 May 2022 15:21:12 -0500 Subject: [PATCH 1/2] Removed gen_energy_vec funtion, Added averaging quantification e_cal_ratio for multiple standards --- CMakeLists.txt | 12 +- src/core/process_whole.cpp | 11 +- src/data_struct/detector.cpp | 23 +- src/data_struct/spectra.cpp | 11 - src/data_struct/spectra.h | 2 - src/io/file/aps/aps_calibration_io.cpp | 426 ------------------------- src/io/file/aps/aps_calibration_io.h | 91 ------ src/io/file/hdf5_io.cpp | 8 +- vcpkg | 2 +- 9 files changed, 35 insertions(+), 551 deletions(-) delete mode 100644 src/io/file/aps/aps_calibration_io.cpp delete mode 100644 src/io/file/aps/aps_calibration_io.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dec16ca2..7285996a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -276,11 +276,12 @@ set(XRF_IO_HEADERS src/core/mem_info.h src/support/mdautils-1.4.1/mda-load.h src/io/file/mda_io.h - src/io/file/mca_io.h + src/io/file/mca_io.h src/io/file/hdf5_io.h src/io/file/netcdf_io.h src/io/file/csv_io.h src/io/file/aps/aps_fit_params_import.h + src/io/file/aps/aps_roi_io.h src/io/file/hl_file_io.h src/io/net/basic_serializer.h src/workflow/source.h @@ -292,13 +293,13 @@ set(XRF_IO_HEADERS src/workflow/xrf/detector_sum_spectra_source.h src/workflow/xrf/spectra_stream_saver.h src/workflow/xrf/spectra_net_streamer.h - src/core/process_streaming.h - src/core/process_whole.h - ) + src/core/process_streaming.h + src/core/process_whole.h + ) set(XRF_IO_SOURCE ${VISUAL_INC} - src/core/mem_info.cpp + src/core/mem_info.cpp src/support/zmq/zmq.hpp src/support/mdautils-1.4.1/mda_loader.c src/io/file/mda_io.cpp @@ -307,6 +308,7 @@ set(XRF_IO_SOURCE src/io/file/netcdf_io.cpp src/io/file/csv_io.cpp src/io/file/aps/aps_fit_params_import.cpp + src/io/file/aps/aps_roi_io.cpp src/io/file/hl_file_io.cpp src/io/net/basic_serializer.cpp src/workflow/xrf/spectra_file_source.cpp diff --git a/src/core/process_whole.cpp b/src/core/process_whole.cpp index d3f1c221..9c43ef13 100644 --- a/src/core/process_whole.cpp +++ b/src/core/process_whole.cpp @@ -144,7 +144,7 @@ bool optimize_integrated_fit_params(data_struct::Analysis_Job* analysis_job, if (params_override != nullptr) { - //load the quantification standard dataset + //load the standard dataset if (false == io::load_and_integrate_spectra_volume(analysis_job->dataset_directory, dataset_filename, detector_num, &int_spectra, params_override)) { logE << "In optimize_integrated_dataset loading dataset" << dataset_filename << " for detector" << detector_num << "\n"; @@ -332,7 +332,7 @@ void proc_spectra(data_struct::Spectra_Volume* spectra_volume, start = std::chrono::system_clock::now(); if (override_params->elements_to_fit.size() < 1) { - logE<<"No elements to fit. Check maps_fit_parameters_override.txt0 - 3 exist"<<"\n"; + logE<<"No elements to fit. Check maps_fit_parameters_override.txt0 - 7 exist"<<"\n"; continue; } @@ -848,13 +848,6 @@ bool perform_quantification(data_struct::Analysis_Job* analysis_job) { fitting::routines::Base_Fit_Routine *fit_routine = fit_itr.second; - // update e_cal_ratio for elements in standards by average value - for (auto& s_itr : detector->quantification_standards) - { - Quantification_Standard* quantification_standard = &(s_itr.second); - detector->update_element_quants(fit_itr.first, quant_itr.first, quantification_standard, &quantification_model, quant_itr.second); - } - logI << Fitting_Routine_To_Str.at(fit_itr.first) << " "<< quant_itr.first << "\n"; Fit_Parameters fit_params; fit_params.add_parameter(Fit_Param("quantifier", 0.0, std::numeric_limits::max(), 1.0, 0.1, E_Bound_Type::FIT)); diff --git a/src/data_struct/detector.cpp b/src/data_struct/detector.cpp index 2a28e3b7..5f26b5f8 100644 --- a/src/data_struct/detector.cpp +++ b/src/data_struct/detector.cpp @@ -171,8 +171,27 @@ void Detector::update_element_quants(Fitting_Routines routine, real_t counts = standard->element_counts.at(routine).at(name); real_t e_cal_factor = (eq_itr.weight * (ic_quantifier)); - real_t e_cal = e_cal_factor / counts; - eq_itr.e_cal_ratio = (real_t)1.0 / e_cal; + if (counts > 0.) + { + real_t e_cal = e_cal_factor / counts; + if (eq_itr.e_cal_ratio == 0.) + { + eq_itr.e_cal_ratio = (real_t)1.0 / e_cal; + } + else // else avg the two + { + real_t second_cal_ratio = (real_t)1.0 / e_cal; + eq_itr.e_cal_ratio += second_cal_ratio; + eq_itr.e_cal_ratio *= 0.5; + } + } + else + { + if (eq_itr.e_cal_ratio == 0.) + { + eq_itr.e_cal_ratio = 1.0e-10; + } + } } } } diff --git a/src/data_struct/spectra.cpp b/src/data_struct/spectra.cpp index 0f442dc2..10117cf7 100644 --- a/src/data_struct/spectra.cpp +++ b/src/data_struct/spectra.cpp @@ -238,16 +238,5 @@ ArrayXr snip_background(const Spectra* const spectra, } -void gen_energy_vector(real_t number_channels, real_t energy_offset, real_t energy_slope, std::vector *out_vec) -{ - - out_vec->resize(number_channels); - for(int i=0; i *out_vec); - typedef std::function IO_Callback_Func_Def; diff --git a/src/io/file/aps/aps_calibration_io.cpp b/src/io/file/aps/aps_calibration_io.cpp deleted file mode 100644 index 6aa1e4d8..00000000 --- a/src/io/file/aps/aps_calibration_io.cpp +++ /dev/null @@ -1,426 +0,0 @@ -/*** -Copyright (c) 2016, UChicago Argonne, LLC. All rights reserved. - -Copyright 2016. UChicago Argonne, LLC. This software was produced -under U.S. Government contract DE-AC02-06CH11357 for Argonne National -Laboratory (ANL), which is operated by UChicago Argonne, LLC for the -U.S. Department of Energy. The U.S. Government has rights to use, -reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR -UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR -ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is -modified to produce derivative works, such modified software should -be clearly marked, so as not to confuse it with the version available -from ANL. - -Additionally, redistribution and use in source and binary forms, with -or without modification, are permitted provided that the following -conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of UChicago Argonne, LLC, Argonne National - Laboratory, ANL, the U.S. Government, nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago -Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -***/ - -/// Initial Author <2016>: Arthur Glowacki - - - -#include "aps_calibration_io.h" - -#include -#include -#include -#include -#include -#include - -#include "mda_io.h" -#include "spectra_volume.h" - -namespace io -{ -namespace file -{ -namespace aps -{ - -APS_Calibration_IO::APS_Calibration_IO() -{ - - - -} - -APS_Calibration_IO::~APS_Calibration_IO() -{ - - -} - -bool APS_Calibration_IO::load(std::string path, std::string standard_filename, data_struct::xrf::Calibration_Standard* calibration, size_t detector_num) -{ - - bool status = _load_standards_txt(path+standard_filename, calibration); - - if (!status) - { - //TODO: log error - std::cout<<"error"; - return status; - } - - // check if standard_file is mca ext - _standard_filename.erase(std::remove_if(_standard_filename.begin(), _standard_filename.end(), ::isspace), _standard_filename.end()); - if(_standard_filename.substr(_standard_filename.find_last_of(".") + 1) == "mca") - { - - std::ifstream standardFileStream(path+_standard_filename); - if (standardFileStream.good()) - { - status = _load_mca(path+_standard_filename, 0, calibration); - } - else - { - std::cout<<"Could not find standard file "<<_standard_filename< element_names; - std::vector element_weights; - - try - { - - for (std::string line; std::getline(paramFileStream, line); ) - //while(std::getline(paramFileStream, line)) - { - std::istringstream strstream(line); - std::getline(strstream, tag, ':'); - //std::cout<<"tag : "<append_element_weight(element_names[i], element_weights[i]); - } - } - else - { - std::cout<<"Error: number of element names ["<resize(num_channels); - - } - else if (tag == "REAL_TIME") - { - std::getline(strstream, str_value, ':'); - spectra->real_time(std::stoi(str_value)); - } - else if (tag == "LIVE_TIME") - { - std::getline(strstream, str_value, ':'); - spectra->live_time(std::stoi(str_value)); - } - else if (tag == "CAL_OFFSET") - { - std::getline(strstream, str_value, ':'); - detector->energy_offset(std::stoi(str_value)); - } - else if (tag == "CAL_SLOPE") - { - std::getline(strstream, str_value, ':'); - detector->energy_slope(std::stoi(str_value)); - } - else if (tag == "CAL_QUAD") - { - std::getline(strstream, str_value, ':'); - detector->energy_quadratic(std::stoi(str_value)); - } - else if (tag == "ENVIRONMENT") - { - /* - if (etag == "S:SRcurrentAI") - calibration->current(std::stoi(str_value)); - elif etag == '2xfm:scaler1_cts1.B': - if IC_US == 0 : - IC_US = real_t(temp) - elif etag == '2xfm:scaler1_cts1.C': - if IC_DS == 0 : - IC_DS = real_t(temp) - elif etag == '2xfm:scaler3_cts1.B': - IC_US = real_t(temp) - elif etag == '2xfm:scaler3_cts1.C': - IC_DS = real_t(temp) - elif etag == '2idd:scaler1_cts1.C': - IC_US = real_t(temp) - elif etag == '2idd:scaler1_cts1.B': - IC_DS = real_t(temp) - elif etag == '8bmb:3820:scaler1_cts1.B': - IC_US = real_t(temp) - elif etag == '8bmb:3820:scaler1_cts1.C': - IC_DS = real_t(temp) - elif etag[5:] == 'A1sens_num.VAL': - amp[0, 0] = real_t(temp) - elif etag[5:] == 'A2sens_num.VAL': - amp[1, 0] = real_t(temp) - elif etag[5:] == 'A3sens_num.VAL': - amp[2, 0] = real_t(temp) - elif etag[5:] == 'A4sens_num.VAL': - amp[3, 0] = real_t(temp) - elif etag[5:] == 'A1sens_unit.VAL': - if (temp == "nA/V") or (temp == "pA/V") or (temp == "uA/V") or (temp == "mA/V"): - if (temp == "pA/V") : amp[0, 1] = 0 - if (temp == "nA/V") : amp[0, 1] = 1 - if (temp == "uA/V") : amp[0, 1] = 2 - if (temp == "mA/V") : amp[0, 1] = 3 - else: - amp[0, 1] = real_t(temp) - elif etag[5:] == 'A2sens_unit.VAL': - if (temp == "nA/V") or (temp == "pA/V") or (temp == "uA/V") or (temp == "mA/V"): - if (temp == "pA/V") : amp[1, 1] = 0 - if (temp == "nA/V") : amp[1, 1] = 1 - if (temp == "uA/V") : amp[1, 1] = 2 - if (temp == "mA/V") : amp[1, 1] = 3 - else: - amp[1, 1] = real_t(temp) - elif etag[5:] == 'A3sens_unit.VAL': - if (temp == "nA/V") or (temp == "pA/V") or (temp == "uA/V") or (temp == "mA/V"): - if (temp == "pA/V") : amp[2, 1] = 0 - if (temp == "nA/V") : amp[2, 1] = 1 - if (temp == "uA/V") : amp[2, 1] = 2 - if (temp == "mA/V") : amp[2, 1] = 3 - else: - amp[2, 1] = real_t(temp) - elif etag[5:] == 'A4sens_unit.VAL': - if (temp == "nA/V") or (temp == "pA/V") or (temp == "uA/V") or (temp == "mA/V") : - if (temp == "pA/V") : amp[3, 1] = 0 - if (temp == "nA/V") : amp[3, 1] = 1 - if (temp == "uA/V") : amp[3, 1] = 2 - if (temp == "mA/V") : amp[3, 1] = 3 - else: - amp[3, 1] = real_t(temp) - */ - } - else if (tag == "DATA") - { - for(size_t i=0; i: Arthur Glowacki - - - -#ifndef APS_CALIBRATION_IO -#define APS_CALIBRATION_IO - -#include "defines.h" -#include "calibration_standard.h" -#include "detector.h" -#include - -namespace io -{ -namespace file -{ -namespace aps -{ - -class DLL_EXPORT APS_Calibration_IO -{ -public: - - APS_Calibration_IO(); - - ~APS_Calibration_IO(); - - bool load(std::string path, std::string standard_filename, data_struct::xrf::Calibration_Standard* calibration, size_t detector_num=0); - -private: - - bool _load_standards_txt(std::string path, data_struct::xrf::Calibration_Standard* calibration); - - bool _load_mca(std::string path, int detector_num, data_struct::xrf::Detector* detector, data_struct::xrf::Spectra* spectra); - - bool _load_mda(std::string path, data_struct::xrf::Calibration_Standard* calibration); - - std::string _standard_filename; - -}; - -}// end namespace aps -}// end namespace file -}// end namespace io - -#endif // APS_Calibration_IO diff --git a/src/io/file/hdf5_io.cpp b/src/io/file/hdf5_io.cpp index 29e68c99..f8fb5f16 100644 --- a/src/io/file/hdf5_io.cpp +++ b/src/io/file/hdf5_io.cpp @@ -4502,8 +4502,8 @@ bool HDF5_IO::save_energy_calib(int spectra_size, real_t energy_offset, real_t e hsize_t count[1] = { 1 }; //save energy vector - std::vector out_vec; - data_struct::gen_energy_vector(spectra_size, energy_offset, energy_slope, &out_vec); + data_struct::ArrayXr energy = data_struct::ArrayXr::LinSpaced(spectra_size, 0, spectra_size-1); + data_struct::ArrayXr ev = energy_offset + (energy * energy_slope) + (Eigen::pow(energy, (real_t)2.0) * energy_quad); if (false == _open_or_create_group(STR_MAPS, _cur_file_id, maps_grp_id)) { @@ -4515,14 +4515,14 @@ bool HDF5_IO::save_energy_calib(int spectra_size, real_t energy_offset, real_t e return false; } - count[0] = out_vec.size(); + count[0] = ev.size(); _create_memory_space(1, count, memoryspace_id); if (false == _open_h5_dataset(STR_ENERGY, H5T_INTEL_R, spec_grp_id, 1, count, count, dset_id, dataspace_id)) { return false; } - status = H5Dwrite(dset_id, H5T_NATIVE_REAL, memoryspace_id, dataspace_id, H5P_DEFAULT, (void*)&out_vec[0]); + status = H5Dwrite(dset_id, H5T_NATIVE_REAL, memoryspace_id, dataspace_id, H5P_DEFAULT, ev.data()); if (status < 0) { logE << " H5Dwrite failed to write " << STR_ENERGY << "\n"; diff --git a/vcpkg b/vcpkg index a106de33..9fb1585e 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit a106de33bbee694e3be6243718aa2a549a692832 +Subproject commit 9fb1585e7a375cfa873697557ced116dfacbd1d7 From d9db0d001fe3c5fba734646ad61c9f89708f8552 Mon Sep 17 00:00:00 2001 From: Arthur Glowacki Date: Mon, 9 May 2022 08:19:08 -0500 Subject: [PATCH 2/2] Fixed netcdf not setting time and counts for loading spectra volume --- CMakeLists.txt | 2 -- src/io/file/mda_io.cpp | 8 ++++---- src/io/file/netcdf_io.cpp | 13 ++++++++++--- vcpkg | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7285996a..9a495187 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -281,7 +281,6 @@ set(XRF_IO_HEADERS src/io/file/netcdf_io.h src/io/file/csv_io.h src/io/file/aps/aps_fit_params_import.h - src/io/file/aps/aps_roi_io.h src/io/file/hl_file_io.h src/io/net/basic_serializer.h src/workflow/source.h @@ -308,7 +307,6 @@ set(XRF_IO_SOURCE src/io/file/netcdf_io.cpp src/io/file/csv_io.cpp src/io/file/aps/aps_fit_params_import.cpp - src/io/file/aps/aps_roi_io.cpp src/io/file/hl_file_io.cpp src/io/net/basic_serializer.cpp src/workflow/xrf/spectra_file_source.cpp diff --git a/src/io/file/mda_io.cpp b/src/io/file/mda_io.cpp index 94355ad1..9945fa3d 100644 --- a/src/io/file/mda_io.cpp +++ b/src/io/file/mda_io.cpp @@ -246,8 +246,8 @@ bool MDA_IO::load_spectra_volume(std::string path, if (_mda_file->header->data_rank == 2) { - logI<<" requested cols "<< _mda_file->header->dimensions[0] << " requested rows " << _mda_file->header->dimensions[1] << - " acquired cols "<< _mda_file->scan->last_point << " acquired rows " << _mda_file->scan->sub_scans[0]->last_point <<"\n"; + logI<<" requested rows "<< _mda_file->header->dimensions[0] << " requested cols " << _mda_file->header->dimensions[1] << + " acquired rows "<< _mda_file->scan->last_point << " acquired cols " << _mda_file->scan->sub_scans[0]->last_point <<"\n"; if(hasNetCDF) { @@ -761,8 +761,8 @@ bool MDA_IO::load_integrated_spectra(std::string path, if (_mda_file->header->data_rank == 2) { - logI << " requested cols " << _mda_file->header->dimensions[0] << " requested rows " << _mda_file->header->dimensions[1] << - " acquired cols " << _mda_file->scan->last_point << " acquired rows " << _mda_file->scan->sub_scans[0]->last_point << "\n"; + logI << " requested rows " << _mda_file->header->dimensions[0] << " requested cols " << _mda_file->header->dimensions[1] << + " acquired rows " << _mda_file->scan->last_point << " acquired cols " << _mda_file->scan->sub_scans[0]->last_point << "\n"; if (hasNetCDF) { diff --git a/src/io/file/netcdf_io.cpp b/src/io/file/netcdf_io.cpp index f91b4c57..7b4756d3 100644 --- a/src/io/file/netcdf_io.cpp +++ b/src/io/file/netcdf_io.cpp @@ -318,15 +318,22 @@ size_t NetCDF_IO::_load_spectra(E_load_type ltype, { output_counts = ((float)ii) / elapsed_realtime; (*spec_line)[j].output_counts(output_counts); - - // recalculate elapsed lifetime - (*spec_line)[j].recalc_elapsed_livetime(); } else if(ltype == E_load_type::INTEGRATED) { output_counts += ((float)ii) / elapsed_realtime; } + if (ltype == E_load_type::LINE || ltype == E_load_type::CALLBACKF) + { + (*spec_line)[j].elapsed_livetime(elapsed_livetime); + (*spec_line)[j].elapsed_realtime(elapsed_realtime); + (*spec_line)[j].input_counts(input_counts); + (*spec_line)[j].output_counts(output_counts); + // recalculate elapsed lifetime + (*spec_line)[j].recalc_elapsed_livetime(); + } + start[2] += header_size + (spectra_size * detector); count[2] = spectra_size; diff --git a/vcpkg b/vcpkg index 9fb1585e..a106de33 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 9fb1585e7a375cfa873697557ced116dfacbd1d7 +Subproject commit a106de33bbee694e3be6243718aa2a549a692832