From 5b5edc3196f3b250f4d246cca1b18a472f875bd3 Mon Sep 17 00:00:00 2001 From: gwd213 Date: Thu, 28 Nov 2024 00:57:44 -0500 Subject: [PATCH] update on script and calib README added --- general_codes/Jaein/QA_database/README.md | 128 +++++++++++++----- .../Jaein/QA_database/README_Calib.md | 91 +++++++++++++ general_codes/Jaein/QA_database/make_plots.py | 85 ++++++++---- .../Jaein/QA_database/put_in_database.py | 25 +++- 4 files changed, 255 insertions(+), 74 deletions(-) create mode 100644 general_codes/Jaein/QA_database/README_Calib.md diff --git a/general_codes/Jaein/QA_database/README.md b/general_codes/Jaein/QA_database/README.md index 3b7053f..436d412 100644 --- a/general_codes/Jaein/QA_database/README.md +++ b/general_codes/Jaein/QA_database/README.md @@ -1,37 +1,91 @@ -# Database Scripts - -This directory contains scripts for processing and analyzing data related to the sPHENIX psql database especially for INTT QA Database. Below are descriptions of the main scripts: -- main Author : Takahiro Kikuchi(Rikkyo Univ.), Jaein Hwang(Korea Univ.) -## put_in_database.py - -This script processes run files and inserts relevant data into the database. - -### Functions - -- `get_run_events()`: Queries the database for run events of type 'physics'. -- `insert_data(runnum, dead_count, runtime, bco_stdev, bco_peak)`: Inserts processed data into the `intt_qa_expert` table. -- `process_run_file(hot_file_path)`: Processes a BADMAP CDB file to count dead, cold, and hot channels. -- `process_bco_file(bco_file_path)`: Processes a BCO CDB file to calculate the standard deviation and peak of BCO differences. -- `calculate_runtime(brtimestamp, ertimestamp)`: Calculates the runtime of a run in minutes. -- `main()`: Main function that orchestrates the processing of run files and insertion of data into the database. - - -## make_plots.py -This script generates various histograms and a tree from the database data related to INTT QA Database. - - ### Functions -- `get_run_events()`: Queries the database for run events. -get_num_events_from_daq(runnumbers): Queries the database for the number of events from DAQ. -- `process_run_file(hot_file_path)`: Processes a run file to count dead, cold, and hot channels. -create_histograms(): Creates histograms for no hit channels, runtime, BCO peak standard deviation, and BCO peak position. -- `create_histograms_with_event_count()`: Creates histograms with event count for no hit channels, runtime, BCO peak standard deviation, and BCO peak position. -- `create_canvas()`: Creates canvases for drawing histograms. -- `create_tree()`: Creates a ROOT tree and branches for run data. -- `fill_histograms_and_tree(rows, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak): Fills histograms and tree with data from the database. -- `fill_histo_with_numevents(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak)`: Fills histograms with event count data. -- ` save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree)` : Saves histograms and tree to a ROOT file. - -### Usage - -Run the script using Python: -python3 make_plots.py +# InttCalib Module + +The `InttCalib` class is a main of the calibration module for the INTT detector. It inherits from `SubsysReco` and provides methods for initializing, processing events, and finalizing the calibration process. + +## Class: InttCalib + +### Constructor +- `InttCalib(std::string const& name = "InttCalib")` + - Initializes the `InttCalib` object with the given name. + +### Destructor +- `~InttCalib()` + - Cleans up dynamically allocated memory. + +### Methods + +#### Initialization and Run Control +- `int InitRun(PHCompositeNode* topNode) override` + - Initializes the run, setting up necessary data structures and loading maps from the CDB. + +- `int process_event(PHCompositeNode* topNode) :woverride` + - Processes each event, updating hitmaps and handling BCO offsets. + +- `int EndRun(int const run_number) override` + - Finalizes the run, generating hot maps and BCO maps. + +#### OutPutFile Setup +- `void SetHotMapCdbFile(std::string const& file)` +- `void SetHotMapPngFile(std::string const& file)` +- `void SetBcoMapCdbFile(std::string const& file)` +- `void SetBcoMapPngFile(std::string const& file)` + +#### Configuration +- `void SetStreamingMode(bool mode)` + - Set Streaming mode. Especially important for BCO Calibration in pp runs +- `void SetBcoMaximumEvent(int mext)` + - Set Maxmium event to produce the BCO maps. +- `void SetRunNumber(int runnum)` + - RunNumber Setting +- `void SetDoFeebyFee(bool in)` + - true : Doing fitting fee by fee for accurate cold channel determination (Default : false) +- `void SetColdSigmaCut(double in)` + - Sigma cut for cold channel +- `void SetHotSigmaCut(double in)` + - Sigma cut for hot channel +- `void SetppMode(bool mode)` + - Not used. Keeping for possibility of future implementation. + +#### Hot Map Generation / FELIX server by server +- `int ConfigureHotMap_v3()` +- `int MakeHotMapCdb_v3()` +- `int MakeHotMapPng_v3()` + +#### Hot Map generation / Fee by fee (fee = 1 half ladder) +- `int ConfigureHotMap_fee()` +- `int MakeHotMapCdb_fee()` +- `int MakeHotMapROOT_fee()` + +#### BCO Map Generation +- `int ConfigureBcoMap()` +- `int MakeBcoMapCdb()` +- `int MakeBcoMapPng()` + +#### Histogram Configuration +- `int ConfigureHist_v3(TH1D*& hist, TF1*& fit, double maxbin, std::map const& hitrate_map, std::string const& name, std::string const& title)` + +#### Old versions (not used) +- `int ConfigureHotMap_v2()` +- `int MakeHotMapCdb_v2()` +- `int MakeHotMapPng_v2()` +- `int ConfigureHotMap()` +- `int MakeHotMapCdb()` +- `int MakeHotMapPng()` +- `int ConfigureHist(TH1D*& hist, TF1*& fit, std::map const& hitrate_map, std::string const& name, std::string const& title)` +- `int ConfigureHist_v2(TH1D*& hist, TF1*& fit, std::map const& hitrate_map, std::string const& name, std::string const& title)` + + + +#### Utility Methods +- `int adjust_hitrate(InttMap::Offline_s const& ofl, double& hitrate) const` +- `int GetIndex(InttMap::RawData_s const& raw, InttMap::Offline_s const& ofl) const` +- `int GetFeeIndex(InttMap::RawData_s const& raw, InttMap::Offline_s const& ofl) const` +- `std::pair CalculateStandardDeviation(const std::vector& data)` +- `Color_t GetFeeColor(int fee) const` + +#### Debugging and Data Management +- `void Debug()` +- `int SaveHitrates()` +- `int LoadHitrates()` + +This class provides a comprehensive set of methods for calibrating the INTT detector, including handling hot and cold channels, generating maps, and configuring histograms. diff --git a/general_codes/Jaein/QA_database/README_Calib.md b/general_codes/Jaein/QA_database/README_Calib.md new file mode 100644 index 0000000..436d412 --- /dev/null +++ b/general_codes/Jaein/QA_database/README_Calib.md @@ -0,0 +1,91 @@ +# InttCalib Module + +The `InttCalib` class is a main of the calibration module for the INTT detector. It inherits from `SubsysReco` and provides methods for initializing, processing events, and finalizing the calibration process. + +## Class: InttCalib + +### Constructor +- `InttCalib(std::string const& name = "InttCalib")` + - Initializes the `InttCalib` object with the given name. + +### Destructor +- `~InttCalib()` + - Cleans up dynamically allocated memory. + +### Methods + +#### Initialization and Run Control +- `int InitRun(PHCompositeNode* topNode) override` + - Initializes the run, setting up necessary data structures and loading maps from the CDB. + +- `int process_event(PHCompositeNode* topNode) :woverride` + - Processes each event, updating hitmaps and handling BCO offsets. + +- `int EndRun(int const run_number) override` + - Finalizes the run, generating hot maps and BCO maps. + +#### OutPutFile Setup +- `void SetHotMapCdbFile(std::string const& file)` +- `void SetHotMapPngFile(std::string const& file)` +- `void SetBcoMapCdbFile(std::string const& file)` +- `void SetBcoMapPngFile(std::string const& file)` + +#### Configuration +- `void SetStreamingMode(bool mode)` + - Set Streaming mode. Especially important for BCO Calibration in pp runs +- `void SetBcoMaximumEvent(int mext)` + - Set Maxmium event to produce the BCO maps. +- `void SetRunNumber(int runnum)` + - RunNumber Setting +- `void SetDoFeebyFee(bool in)` + - true : Doing fitting fee by fee for accurate cold channel determination (Default : false) +- `void SetColdSigmaCut(double in)` + - Sigma cut for cold channel +- `void SetHotSigmaCut(double in)` + - Sigma cut for hot channel +- `void SetppMode(bool mode)` + - Not used. Keeping for possibility of future implementation. + +#### Hot Map Generation / FELIX server by server +- `int ConfigureHotMap_v3()` +- `int MakeHotMapCdb_v3()` +- `int MakeHotMapPng_v3()` + +#### Hot Map generation / Fee by fee (fee = 1 half ladder) +- `int ConfigureHotMap_fee()` +- `int MakeHotMapCdb_fee()` +- `int MakeHotMapROOT_fee()` + +#### BCO Map Generation +- `int ConfigureBcoMap()` +- `int MakeBcoMapCdb()` +- `int MakeBcoMapPng()` + +#### Histogram Configuration +- `int ConfigureHist_v3(TH1D*& hist, TF1*& fit, double maxbin, std::map const& hitrate_map, std::string const& name, std::string const& title)` + +#### Old versions (not used) +- `int ConfigureHotMap_v2()` +- `int MakeHotMapCdb_v2()` +- `int MakeHotMapPng_v2()` +- `int ConfigureHotMap()` +- `int MakeHotMapCdb()` +- `int MakeHotMapPng()` +- `int ConfigureHist(TH1D*& hist, TF1*& fit, std::map const& hitrate_map, std::string const& name, std::string const& title)` +- `int ConfigureHist_v2(TH1D*& hist, TF1*& fit, std::map const& hitrate_map, std::string const& name, std::string const& title)` + + + +#### Utility Methods +- `int adjust_hitrate(InttMap::Offline_s const& ofl, double& hitrate) const` +- `int GetIndex(InttMap::RawData_s const& raw, InttMap::Offline_s const& ofl) const` +- `int GetFeeIndex(InttMap::RawData_s const& raw, InttMap::Offline_s const& ofl) const` +- `std::pair CalculateStandardDeviation(const std::vector& data)` +- `Color_t GetFeeColor(int fee) const` + +#### Debugging and Data Management +- `void Debug()` +- `int SaveHitrates()` +- `int LoadHitrates()` + +This class provides a comprehensive set of methods for calibrating the INTT detector, including handling hot and cold channels, generating maps, and configuring histograms. diff --git a/general_codes/Jaein/QA_database/make_plots.py b/general_codes/Jaein/QA_database/make_plots.py index 386f6ee..0bc64eb 100644 --- a/general_codes/Jaein/QA_database/make_plots.py +++ b/general_codes/Jaein/QA_database/make_plots.py @@ -17,7 +17,7 @@ def get_run_events(): ) with pyodbc.connect(connection_string) as conn: cursor = conn.cursor() - cursor.execute('SELECT runnumber, nohitch, runtime, bco_stdev, bco_peak from intt_qa_expert;') + cursor.execute('SELECT runnumber, deadch, runtime, bco_stddev, bco_peak, hitrate0, hitrate1, hitrate2, hitrate3, hitrate4, hitrate5, hitrate6, hitrate7 from intt_qa_expert;') return cursor.fetchall() def get_num_events_from_daq(runnumbers): @@ -64,7 +64,8 @@ def create_histograms(): hist_runtime = ROOT.TH1D("hist_runtime", "Runtime", 66, 0, 65) hist_bco_stdev = ROOT.TH1D("hist_bco_stdev", "BCO peak StdDev", 100, 0, 100) hist_bco_peak = ROOT.TH1D("hist_bco_peak", "BCO peak position", 128, 0, 127) - return hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak + hist_hitrate = [ROOT.TH1D(f"hist_hitrate{i}", f"Hitrate {i}", 100, 0, 1) for i in range(8)] + return hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, hist_hitrate def create_histograms_with_event_count(): hist_nohitch = ROOT.TH1D("hist_nohitch_event_count", "Number of no hit channel with event count", 50000, 0, 1000000) @@ -89,28 +90,49 @@ def create_tree(): _nohitch = np.ndarray(1, 'int') _bco_stdev = np.ndarray(1, 'double') _bco_peak = np.ndarray(1, 'int') + _events = np.ndarray(1, 'int') + _hitrate = [np.ndarray(1, 'double') for _ in range(8)] tree.Branch("runnumber", _runnum, "runnumber/I") tree.Branch("runtime", _runtime, "runtime/I") tree.Branch("nohitch", _nohitch, "nohitch/I") tree.Branch("bco_stdev", _bco_stdev, "bco_stdev/D") tree.Branch("bco_peak", _bco_peak, "bco_peak/I") - return tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak + tree.Branch("events", _events, "events/I") + for i in range(8): + tree.Branch(f"hitrate{i}", _hitrate[i], f"hitrate{i}/D") + return tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak, _events, _hitrate -def fill_histograms_and_tree(rows, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak): - for i in range(6, len(rows)): - runnum, nohitch, runtime, bco_stdev, bco_peak = rows[i] +def create_graphs(): + graphs = [ROOT.TGraph() for _ in range(8)] + for i, graph in enumerate(graphs): + graph.SetNameTitle(f"graph_hitrate{i}", f"Hitrate {i} vs Runnumber") + graph.GetXaxis().SetTitle("Runnumber") + graph.GetYaxis().SetTitle(f"Hitrate {i}") + return graphs + +def fill_histograms_and_tree(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, hist_hitrate, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak, _events, _hitrate, graphs): + event_count_dict = {runnumber: event_count for runnumber, event_count in num_events} + for i in range(len(rows)): + runnum, nohitch, runtime, bco_stdev, bco_peak, *hitrates = rows[i] + event_count = event_count_dict.get(runnum, 1) _runnum[0] = runnum _nohitch[0] = nohitch _runtime[0] = runtime _bco_peak[0] = bco_peak _bco_stdev[0] = bco_stdev - hist_nohitch.Fill(nohitch, 1) - hist_nohitch_zoom.Fill(nohitch, 1) - hist_runtime.Fill(runtime, 1) - hist_bco_stdev.Fill(bco_stdev, 1) - hist_bco_peak.Fill(bco_peak, 1) + _events[0] = event_count + for j in range(8): + _hitrate[j][0] = hitrates[j] + hist_nohitch.Fill(nohitch, event_count) + hist_nohitch_zoom.Fill(nohitch, event_count) + hist_runtime.Fill(runtime, event_count) + hist_bco_stdev.Fill(bco_stdev, event_count) + hist_bco_peak.Fill(bco_peak, event_count) + for j in range(8): + hist_hitrate[j].Fill(hitrates[j], event_count) + graphs[j].SetPoint(i, runnum, hitrates[j]) tree.Fill() - print(runnum, nohitch, runtime, bco_stdev, bco_peak) + print(runnum, nohitch, runtime, bco_stdev, bco_peak, event_count, *hitrates) def fill_histo_with_numevents(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak): event_count_dict = {runnumber: event_count for runnumber, event_count in num_events} @@ -124,7 +146,7 @@ def fill_histo_with_numevents(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_bco_peak.Fill(bco_peak, event_count) print(runnum, nohitch, runtime, bco_stdev, bco_peak, event_count) -def save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree): +def save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, hist_hitrate, tree): canvas0, canvas00, canvas1, canvas2, canvas3 = create_canvas() canvas0.cd() hist_nohitch.Draw() @@ -141,30 +163,33 @@ def save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist canvas3.cd() hist_bco_peak.Draw() hist_bco_peak.Write() + for i in range(8): + canvas = ROOT.TCanvas(f"canvas_hitrate{i}", f"Hitrate {i} for all run", 800, 600) + canvas.cd() + hist_hitrate[i].Draw() + hist_hitrate[i].Write() + canvas.SaveAs(f"hitrate{i}.png") tree.Write() +def save_graphs(graphs): + for i, graph in enumerate(graphs): + canvas = ROOT.TCanvas(f"canvas_graph_hitrate{i}", f"Hitrate {i} vs Runnumber", 800, 600) + canvas.cd() + graph.Draw("ALP") + canvas.SaveAs(f"graph_hitrate{i}.png") + graph.Write() + def main(): sfile = ROOT.TFile("SaveFile.root", "RECREATE") rows = get_run_events() - hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak = create_histograms() - tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak = create_tree() - fill_histograms_and_tree(rows, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak) - save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree) - - # Extract runnumbers from rows + hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, hist_hitrate = create_histograms() runnumbers = [row[0] for row in rows] - - # New histogram creation and filling with event count num_events = get_num_events_from_daq(runnumbers) - hist_nohitch_event_count, hist_nohitch_zoom_event_count, hist_runtime_event_count, hist_bco_stdev_event_count, hist_bco_peak_event_count = create_histograms_with_event_count() - fill_histo_with_numevents(rows, num_events, hist_nohitch_event_count, hist_nohitch_zoom_event_count, hist_runtime_event_count, hist_bco_stdev_event_count, hist_bco_peak_event_count) - - # Save new histograms - hist_nohitch_event_count.Write() - hist_nohitch_zoom_event_count.Write() - hist_runtime_event_count.Write() - hist_bco_stdev_event_count.Write() - hist_bco_peak_event_count.Write() + tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak, _events, _hitrate = create_tree() + graphs = create_graphs() + fill_histograms_and_tree(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, hist_hitrate, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak, _events, _hitrate, graphs) + save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, hist_hitrate, tree) + save_graphs(graphs) sfile.Close() if __name__ == "__main__": diff --git a/general_codes/Jaein/QA_database/put_in_database.py b/general_codes/Jaein/QA_database/put_in_database.py index fae4d61..7ef63e8 100644 --- a/general_codes/Jaein/QA_database/put_in_database.py +++ b/general_codes/Jaein/QA_database/put_in_database.py @@ -23,7 +23,7 @@ def get_run_events(): cursor.execute("select * from run where runtype = 'physics' order by runnumber;") return cursor.fetchall() -def insert_data(runnum, dead_count, runtime, bco_stdev, bco_peak): +def insert_data(runnum, dead_count, runtime, bco_stdev, bco_peak,hitrate0,hitrate1,hitrate2,hitrate3,hitrate4,hitrate5,hitrate6,hitrate7): """Insert data into the database""" connection_string = ( 'DRIVER={PostgreSQL};' @@ -34,8 +34,8 @@ def insert_data(runnum, dead_count, runtime, bco_stdev, bco_peak): with pyodbc.connect(connection_string) as conn: cursor = conn.cursor() cursor.execute( - "INSERT INTO intt_qa_expert (runnumber, nohitch, runtime, bco_stdev, bco_peak) VALUES (?, ?, ?, ?, ?);", - (runnum, dead_count, runtime, bco_stdev, bco_peak) + "INSERT INTO intt_qa_expert (runnumber, deadch, runtime, bco_stddev, bco_peak, hitrate0,hitrate1,hitrate2,hitrate3,hitrate4,hitrate5,hitrate6,hitrate7) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", + (runnum, dead_count, runtime, bco_stdev, bco_peak,hitrate0,hitrate1,hitrate2,hitrate3,hitrate4,hitrate5,hitrate6,hitrate7) ) conn.commit() @@ -62,7 +62,18 @@ def process_run_file(hot_file_path): return None if not os.path.isfile(hot_file_path): return None - return [Dead, Cold, Hot] + + tree_single = file_hot.Get("Single") + tree_single.GetEntry(0) + Dmean0 = tree_single.Dmean0 + Dmean1 = tree_single.Dmean1 + Dmean2 = tree_single.Dmean2 + Dmean3 = tree_single.Dmean3 + Dmean4 = tree_single.Dmean4 + Dmean5 = tree_single.Dmean5 + Dmean6 = tree_single.Dmean6 + Dmean7 = tree_single.Dmean7 + return [Dead, Cold, Hot, Dmean0, Dmean1, Dmean2, Dmean3, Dmean4, Dmean5, Dmean6, Dmean7] def process_bco_file(bco_file_path): BCO_stdev = 0 @@ -135,10 +146,10 @@ def main(): result = process_run_file(file_paths) bco_result = process_bco_file(bco_file_paths) if result and bco_result: - Dead, Cold, Hot = result + Dead, Cold, Hot, Dmean0 ,Dmean1, Dmean2, Dmean3, Dmean4, Dmean5, Dmean6, Dmean7= result BCO_stdev, BCO_peak = bco_result - BCO_stdev = round(BCO_stdev, 4) - insert_data(runnum, Dead, runtime, BCO_stdev, BCO_peak) + BCO_stdev = round(BCO_stdev, 4) + insert_data(runnum, Dead, runtime, BCO_stdev, BCO_peak,Dmean0,Dmean1,Dmean2,Dmean3,Dmean4,Dmean5,Dmean6,Dmean7) print(runnum) print(f"Dead: {Dead}, Cold: {Cold}, Hot: {Hot}, Runtime: {runtime} minutes, BCO_stdev: {BCO_stdev}, BCO_peak: {BCO_peak}") else: