Skip to content

Commit

Permalink
Merge pull request #143 from aglowacki/master
Browse files Browse the repository at this point in the history 
Added reading v9 roi's
  • Loading branch information
Arthur Glowacki authored Jul 15, 2022
2 parents 310106b + b540ccf commit e9ad207
Show file tree
Hide file tree
Showing 14 changed files with 719 additions and 70 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ set(XRF_IO_HEADERS
src/io/file/netcdf_io.h
src/io/file/csv_io.h
src/io/file/aps/aps_fit_params_import.h
src/io/file/aps/aps_roi.h
src/io/file/file_scan.h
src/io/file/hl_file_io.h
src/io/net/basic_serializer.h
Expand All @@ -299,6 +300,7 @@ set(XRF_IO_SOURCE
src/io/file/netcdf_io.cpp
src/io/file/file_scan.cpp
src/io/file/hl_file_io.cpp
src/io/file/aps/aps_roi.cpp
src/io/net/basic_serializer.cpp
src/workflow/xrf/spectra_file_source.cpp
src/workflow/xrf/spectra_net_source.cpp
Expand Down
6 changes: 3 additions & 3 deletions docs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
#### Description
ROI fitting

### NNLS
### Non-Negative Least Squares (NNLS)

### SVD
### Singualr Value Decomposition (SVD)

### MATRIX
### Per Pixel (MATRIX)

## Quantification
### Option
Expand Down
63 changes: 47 additions & 16 deletions src/core/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ void help()
" 1 = matrix batch fit\n 2 = batch fit without tails\n 3 = batch fit with tails\n 4 = batch fit with free E, everything else fixed \n";
logit_s<<"--optimize-fit-routine : <general,hybrid> General (default): passes elements amplitudes as fit parameters. Hybrid only passes fit parameters and fits element amplitudes using NNLS\n";
logit_s<<"--optimizer <lmfit, mpfit> : Choose which optimizer to use for --optimize-fit-override-params or matrix fit routine \n";
logit_s<<"--optimize-rois : Looks in 'rois' directory and performs --optimize-fit-override-params on each roi separately. \n";
logit_s<<"Fitting Routines: \n";
logit_s<< "--fit <routines,> comma seperated \n";
logit_s<<" roi : element energy region of interest \n";
Expand Down Expand Up @@ -108,6 +109,15 @@ void set_optimizer(Command_Line_Parser& clp, data_struct::Analysis_Job<T_real>&
{
analysis_job.set_optimizer(clp.get_option("--optimizer"));
}

if (clp.option_exists("--optimize-fit-routine"))
{
std::string opt = clp.get_option("--optimize-fit-routine");
if (opt == "hybrid")
{
analysis_job.optimize_fit_routine = OPTIMIZE_FIT_ROUTINE::HYBRID;
}
}
}

// ----------------------------------------------------------------------------
Expand Down Expand Up @@ -212,9 +222,13 @@ template <typename T_real>
void set_fit_routines(Command_Line_Parser& clp, data_struct::Analysis_Job<T_real>& analysis_job)
{
//Look for which analysis types we want to run
if (clp.option_exists("--fit"))
if (clp.option_exists("--fit") || clp.option_exists("--quantify-fit"))
{
std::string fitting = clp.get_option("--fit");
if (fitting.length() < 1)
{
fitting = clp.get_option("--quantify-fit");
}
if (fitting.find(',') != std::string::npos)
{
// if we found a comma, split the string to get list of dataset files
Expand Down Expand Up @@ -438,15 +452,6 @@ int run_optimization(Command_Line_Parser& clp)
}
}

if (clp.option_exists("--optimize-fit-routine"))
{
std::string opt = clp.get_option("--optimize-fit-routine");
if (opt == "hybrid")
{
analysis_job.optimize_fit_routine = OPTIMIZE_FIT_ROUTINE::HYBRID;
}
}

if (io::file::init_analysis_job_detectors(&analysis_job))
{
io::file::File_Scan::inst()->populate_netcdf_hdf5_files(analysis_job.dataset_directory);
Expand Down Expand Up @@ -477,16 +482,16 @@ int run_quantification(Command_Line_Parser& clp)
return -1;
}
set_fit_routines(clp, analysis_job);

if (analysis_job.fitting_routines.size() == 0)
{
logE << "Please specify fit routines with --quantify-fit roi,nnls,matrix \n";
return -1;
}
//Check if we want to quantifiy with a standard
if (clp.option_exists("--quantify-with"))
{
analysis_job.quantification_standard_filename = clp.get_option("--quantify-with");
}
if (clp.option_exists("--quantify-fit"))
{
//TODO: parse save as --fit so we can generate calibration curve without refitting
}

if (io::file::init_analysis_job_detectors(&analysis_job))
{
Expand Down Expand Up @@ -515,6 +520,22 @@ int run_quantification(Command_Line_Parser& clp)

// ----------------------------------------------------------------------------

int run_optimize_rois(Command_Line_Parser& clp)
{
data_struct::Analysis_Job<double> analysis_job;

if (set_general_options(clp, analysis_job) == -1)
{
return -1;
}

optimize_rois(analysis_job);

return 0;
}

// ----------------------------------------------------------------------------

int run_streaming(Command_Line_Parser& clp)
{
data_struct::Analysis_Job<float> analysis_job;
Expand Down Expand Up @@ -713,7 +734,12 @@ int run_h5_file_updates(Command_Line_Parser& clp)
int main(int argc, char* argv[])
{

//std::string whole_command_line = "";
std::string whole_command_line = "";
for (int i = 0; i < argc; i++)
{
whole_command_line += std::string(argv[i]) + " ";
}
logI << whole_command_line << "\n";

//Performance measure
std::chrono::time_point<std::chrono::system_clock> start, end;
Expand Down Expand Up @@ -770,6 +796,11 @@ int main(int argc, char* argv[])
run_quantification(clp);
}

if (clp.option_exists("--optimize-rois"))
{
run_optimize_rois(clp);
}

run_h5_file_updates(clp);

end = std::chrono::system_clock::now();
Expand Down
133 changes: 115 additions & 18 deletions src/core/process_whole.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,23 +53,17 @@ using namespace std::placeholders; //for _1, _2,
// ----------------------------------------------------------------------------

bool optimize_integrated_fit_params(data_struct::Analysis_Job<double> * analysis_job,
std::string dataset_filename,
data_struct::Spectra<double>& int_spectra,
size_t detector_num,
data_struct::Params_Override<double>* params_override,
std::string save_filename,
data_struct::Fit_Parameters<double>& out_fitp)
{
fitting::models::Gaussian_Model<double> model;
bool ret_val = false;
data_struct::Spectra<double> int_spectra;

if (params_override != nullptr)
{
//load the quantification standard dataset
if (false == io::file::load_and_integrate_spectra_volume(analysis_job->dataset_directory, dataset_filename, detector_num, &int_spectra, params_override))
{
logE << "In optimize_integrated_dataset loading dataset" << dataset_filename << " for detector" << detector_num << "\n";
return false;
}
//Range of energy in spectra to fit
fitting::models::Range energy_range = data_struct::get_energy_range<double>(int_spectra.size(), &(params_override->fit_params));

Expand Down Expand Up @@ -124,7 +118,7 @@ bool optimize_integrated_fit_params(data_struct::Analysis_Job<double> * analysis
ret_val = false;
break;
}
io::file::save_optimized_fit_params(analysis_job->dataset_directory, dataset_filename, detector_num, result, &out_fitp, &int_spectra, &(params_override->elements_to_fit));
io::file::save_optimized_fit_params(analysis_job->dataset_directory, save_filename, detector_num, result, &out_fitp, &int_spectra, &(params_override->elements_to_fit));

delete fit_routine;
}
Expand All @@ -141,6 +135,7 @@ void generate_optimal_params(data_struct::Analysis_Job<double>* analysis_job)
std::unordered_map<int, data_struct::Params_Override<double>*> params;
std::unordered_map<int, float> detector_file_cnt;
data_struct::Params_Override<double>* params_override = nullptr;
data_struct::Spectra<double> int_spectra;

std::string full_path = analysis_job->dataset_directory + DIR_END_CHAR + "maps_fit_parameters_override.txt";

Expand Down Expand Up @@ -179,19 +174,27 @@ void generate_optimal_params(data_struct::Analysis_Job<double>* analysis_job)

}

data_struct::Fit_Parameters<double> out_fitp;
if (optimize_integrated_fit_params(analysis_job, itr, detector_num, params_override, out_fitp))
//load the int spectra from the dataset.
if (io::file::load_and_integrate_spectra_volume(analysis_job->dataset_directory, itr, detector_num, &int_spectra, params_override))
{
detector_file_cnt[detector_num] += 1.0;
if (fit_params_avgs.count(detector_num) > 0)
data_struct::Fit_Parameters<double> out_fitp;
if (optimize_integrated_fit_params(analysis_job, int_spectra, detector_num, params_override, itr, out_fitp))
{
fit_params_avgs[detector_num].sum_values(out_fitp);
}
else
{
fit_params_avgs[detector_num] = out_fitp;
detector_file_cnt[detector_num] += 1.0;
if (fit_params_avgs.count(detector_num) > 0)
{
fit_params_avgs[detector_num].sum_values(out_fitp);
}
else
{
fit_params_avgs[detector_num] = out_fitp;
}
}
}
else
{
logE << "In optimize_integrated_dataset loading dataset" << itr << " for detector" << detector_num << "\n";
}
}
}

Expand Down Expand Up @@ -515,6 +518,100 @@ bool perform_quantification(data_struct::Analysis_Job<double>* analysis_job)

}

// ----------------------------------------------------------------------------

void find_and_optimize_roi(data_struct::Analysis_Job<double>& analysis_job, int detector_num, std::map<int, std::vector<std::pair<unsigned int, unsigned int>>>& rois, std::string search_filename)
{
std::vector<std::string> files = io::file::File_Scan::inst()->find_all_dataset_files(analysis_job.dataset_directory + "img.dat", search_filename);
if (files.size() > 0)
{
for (auto& roi_itr : rois)
{
Spectra<double> int_spectra;
std::string file_path = analysis_job.dataset_directory + "img.dat" + DIR_END_CHAR + files[0];
if (io::file::HDF5_IO::inst()->load_integrated_spectra_analyzed_h5_roi(file_path, &int_spectra, roi_itr.second))
{
data_struct::Params_Override<double> params_override;
//load override parameters
if (false == io::file::load_override_params(analysis_job.dataset_directory, detector_num, &params_override))
{
if (false == io::file::load_override_params(analysis_job.dataset_directory, -1, &params_override))
{
logE << "Loading maps_fit_parameters_override.txt\n";
return;
}
}
data_struct::Fit_Parameters<double> out_fitp;
std::string roi_name = std::to_string(roi_itr.first);
if (false == optimize_integrated_fit_params(&analysis_job, int_spectra, detector_num, &params_override, files[0]+ "_roi_" + roi_name, out_fitp))
{
logE << "Failed to optimize ROI "<< file_path<<" : "<< roi_name<<".\n";
}
}
}
}
else
{
logE << "Could not find dataset file for " << search_filename << "\n";
}
}

// ----------------------------------------------------------------------------

void optimize_single_roi(data_struct::Analysis_Job<double>& analysis_job, std::string roi_file_name)
{
//std::map<int, std::vector<std::pair<unsigned int, unsigned int>>> rois;
std::map<int, std::vector<std::pair<unsigned int, unsigned int>>> rois;
std::string search_filename;
//data_struct::Detector<double>* detector;
if (io::file::aps::load_v9_rois(analysis_job.dataset_directory + "rois" + DIR_END_CHAR + roi_file_name, rois))
{
logI << "Loaded roi file "<<roi_file_name << "\n";
//get dataset file number
std::string dataset_num;
std::istringstream strstream(roi_file_name);
std::getline(strstream, dataset_num, '_');
if (dataset_num.length() > 0)
{
// iterate through all
for (size_t detector_num : analysis_job.detector_num_arr)
{
//detector = analysis_job.get_detector(detector_num);
if (detector_num != -1)
{
std::string str_detector_num = std::to_string(detector_num);
search_filename = dataset_num + ".h5" + str_detector_num;
find_and_optimize_roi(analysis_job, detector_num, rois, search_filename);
}
}
//now for the avg h5
//detector = analysis_job.get_detector(-1);
search_filename = dataset_num + ".h5";
dataset_num + ".h5";
find_and_optimize_roi(analysis_job, -1, rois, search_filename);
}
else
{
logE<<"Error parsing dataset number for "<< roi_file_name << "\n";
}

}
else
{
logE << "Error loading roi file "<< roi_file_name << "\n";
}
}

// ----------------------------------------------------------------------------

void optimize_rois(data_struct::Analysis_Job<double>& analysis_job)
{
for (auto& itr : io::file::File_Scan::inst()->find_all_dataset_files(analysis_job.dataset_directory + "rois", ".roi"))
{
optimize_single_roi(analysis_job, itr);
}
}

// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------
7 changes: 7 additions & 0 deletions src/core/process_whole.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ POSSIBILITY OF SUCH DAMAGE.

#include "data_struct/quantification_standard.h"

#include "io/file/aps/aps_roi.h"


using namespace std::placeholders; //for _1, _2,

Expand All @@ -117,6 +119,10 @@ DLL_EXPORT void generate_optimal_params(data_struct::Analysis_Job<double>* analy

void load_and_fit_quatification_datasets(data_struct::Analysis_Job<double>* analysis_job, size_t detector_num, vector<Quantification_Standard<double>>& standard_element_weights, unordered_map<size_t, double>& quant_map);

void optimize_single_roi(data_struct::Analysis_Job<double>& analysis_job, std::string roi_file_name);

DLL_EXPORT void optimize_rois(data_struct::Analysis_Job<double>& analysis_job);

// ----------------------------------------------------------------------------

template<typename T_real>
Expand Down Expand Up @@ -639,6 +645,7 @@ DLL_EXPORT void iterate_datasets_and_update(data_struct::Analysis_Job<T_real>& a
}
}
}

// ----------------------------------------------------------------------------

#endif
Loading

0 comments on commit e9ad207

Please sign in to comment.