From 34a437c942f9424ef750a1640823b3c44bcbeae5 Mon Sep 17 00:00:00 2001 From: gwd213 Date: Tue, 26 Nov 2024 20:04:24 -0500 Subject: [PATCH] INTT QA database code added --- general_codes/Jaein/QA_database/README.md | 37 ++++ general_codes/Jaein/QA_database/make_plots.py | 171 ++++++++++++++++++ .../Jaein/QA_database/put_in_database.py | 153 ++++++++++++++++ 3 files changed, 361 insertions(+) create mode 100644 general_codes/Jaein/QA_database/README.md create mode 100644 general_codes/Jaein/QA_database/make_plots.py create mode 100644 general_codes/Jaein/QA_database/put_in_database.py diff --git a/general_codes/Jaein/QA_database/README.md b/general_codes/Jaein/QA_database/README.md new file mode 100644 index 0000000..43c3b69 --- /dev/null +++ b/general_codes/Jaein/QA_database/README.md @@ -0,0 +1,37 @@ +# Database Scripts + +This directory contains scripts for processing and analyzing data related to the sPHENIX experiment. Below are descriptions of the main scripts: + +## put_in_database.py + +This script processes run files and inserts relevant data into the database. + +### Functions + +- `get_run_events()`: Queries the database for run events of type 'physics'. +- `insert_data(runnum, dead_count, runtime, bco_stdev, bco_peak)`: Inserts processed data into the `intt_qa_expert` table. +- `process_run_file(hot_file_path)`: Processes a run file to count dead, cold, and hot channels. +- `process_bco_file(bco_file_path)`: Processes a BCO file to calculate the standard deviation and peak of BCO differences. +- `calculate_runtime(brtimestamp, ertimestamp)`: Calculates the runtime of a run in minutes. +- `main()`: Main function that orchestrates the processing of run files and insertion of data into the database. + + +### make_plots.py +This script generates various histograms and a tree from the database data related to the sPHENIX experiment. + + ## Functions +- `get_run_events()`: Queries the database for run events. +get_num_events_from_daq(runnumbers): Queries the database for the number of events from DAQ. +- `process_run_file(hot_file_path)`: Processes a run file to count dead, cold, and hot channels. +create_histograms(): Creates histograms for no hit channels, runtime, BCO peak standard deviation, and BCO peak position. +- `create_histograms_with_event_count()`: Creates histograms with event count for no hit channels, runtime, BCO peak standard deviation, and BCO peak position. +- `create_canvas()`: Creates canvases for drawing histograms. +- `create_tree()`: Creates a ROOT tree and branches for run data. +- `fill_histograms_and_tree(rows, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak): Fills histograms and tree with data from the database. +- `fill_histo_with_numevents(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak)`: Fills histograms with event count data. +-` save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree)` : Saves histograms and tree to a ROOT file. + +### Usage + +Run the script using Python: +python3 make_plots.py \ No newline at end of file diff --git a/general_codes/Jaein/QA_database/make_plots.py b/general_codes/Jaein/QA_database/make_plots.py new file mode 100644 index 0000000..386f6ee --- /dev/null +++ b/general_codes/Jaein/QA_database/make_plots.py @@ -0,0 +1,171 @@ +import ROOT +import numpy as np +import pyodbc +import os +import collections +import math + +ROOT.gROOT.SetBatch(True) + +def get_run_events(): + """Query database for run events""" + connection_string = ( + 'DRIVER={PostgreSQL};' + 'SERVER=sphnxdbmaster;' + 'DATABASE=intt;' + 'PORT=5432;' + ) + with pyodbc.connect(connection_string) as conn: + cursor = conn.cursor() + cursor.execute('SELECT runnumber, nohitch, runtime, bco_stdev, bco_peak from intt_qa_expert;') + return cursor.fetchall() + +def get_num_events_from_daq(runnumbers): + """Query database for number of events from daq""" + connection_string = ( + 'DRIVER={PostgreSQL};' + 'SERVER=sphnxdaqdbreplica;' + 'DATABASE=daq;' + 'PORT=5432;' + ) + num_events = [] + with pyodbc.connect(connection_string) as conn: + cursor = conn.cursor() + for runnumber in runnumbers: + cursor.execute("SELECT eventsinrun FROM run WHERE runtype = 'physics' AND runnumber = ?", runnumber) + row = cursor.fetchone() + if row: + num_events.append((runnumber, row[0])) + return num_events + +def process_run_file(hot_file_path): + Dead, Cold, Hot = 0, 0, 0 + if os.path.isfile(hot_file_path): + file_hot = ROOT.TFile.Open(hot_file_path) + tree_hot = file_hot.Get("Multiple") + if tree_hot: + for i in range(tree_hot.GetEntries()): + tree_hot.GetEvent(i) + if tree_hot.Iflag == 1: + Dead += 1 + elif tree_hot.Iflag == 4: + Cold += 1 + elif tree_hot.Iflag == 8: + Hot += 1 + else: + return None + else: + return None + return [Dead, Cold, Hot] + +def create_histograms(): + hist_nohitch = ROOT.TH1D("hist_nohitch", "Number of no hit channel", 50000, 0, 1000000) + hist_nohitch_zoom = ROOT.TH1D("hist_nohitch_zoom", "Number of no hit channel", 100, 0, 30000) + hist_runtime = ROOT.TH1D("hist_runtime", "Runtime", 66, 0, 65) + hist_bco_stdev = ROOT.TH1D("hist_bco_stdev", "BCO peak StdDev", 100, 0, 100) + hist_bco_peak = ROOT.TH1D("hist_bco_peak", "BCO peak position", 128, 0, 127) + return hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak + +def create_histograms_with_event_count(): + hist_nohitch = ROOT.TH1D("hist_nohitch_event_count", "Number of no hit channel with event count", 50000, 0, 1000000) + hist_nohitch_zoom = ROOT.TH1D("hist_nohitch_zoom_event_count", "Number of no hit channel with event count", 100, 0, 30000) + hist_runtime = ROOT.TH1D("hist_runtime_event_count", "Runtime with event count", 66, 0, 65) + hist_bco_stdev = ROOT.TH1D("hist_bco_stdev_event_count", "BCO peak StdDev with event count", 100, 0, 100) + hist_bco_peak = ROOT.TH1D("hist_bco_peak_event_count", "BCO peak position with event count", 128, 0, 127) + return hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak + +def create_canvas(): + canvas0 = ROOT.TCanvas("canvas_nohitch", "Number of no hit channel for all run", 800, 600) + canvas00 = ROOT.TCanvas("canvas_nohitch_zoom", "Number of no hit channel for all run", 800, 600) + canvas1 = ROOT.TCanvas("canvas_runtime", "Run time for all run", 800, 600) + canvas2 = ROOT.TCanvas("canvas_bco_stdev", "BCO peak standard deviation for all run", 800, 600) + canvas3 = ROOT.TCanvas("canvas_bco_peak", "BCO peak position for all run", 800, 600) + return canvas0, canvas00, canvas1, canvas2, canvas3 + +def create_tree(): + tree = ROOT.TTree("tree", "tree") + _runnum = np.ndarray(1, 'int') + _runtime = np.ndarray(1, 'int') + _nohitch = np.ndarray(1, 'int') + _bco_stdev = np.ndarray(1, 'double') + _bco_peak = np.ndarray(1, 'int') + tree.Branch("runnumber", _runnum, "runnumber/I") + tree.Branch("runtime", _runtime, "runtime/I") + tree.Branch("nohitch", _nohitch, "nohitch/I") + tree.Branch("bco_stdev", _bco_stdev, "bco_stdev/D") + tree.Branch("bco_peak", _bco_peak, "bco_peak/I") + return tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak + +def fill_histograms_and_tree(rows, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak): + for i in range(6, len(rows)): + runnum, nohitch, runtime, bco_stdev, bco_peak = rows[i] + _runnum[0] = runnum + _nohitch[0] = nohitch + _runtime[0] = runtime + _bco_peak[0] = bco_peak + _bco_stdev[0] = bco_stdev + hist_nohitch.Fill(nohitch, 1) + hist_nohitch_zoom.Fill(nohitch, 1) + hist_runtime.Fill(runtime, 1) + hist_bco_stdev.Fill(bco_stdev, 1) + hist_bco_peak.Fill(bco_peak, 1) + tree.Fill() + print(runnum, nohitch, runtime, bco_stdev, bco_peak) + +def fill_histo_with_numevents(rows, num_events, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak): + event_count_dict = {runnumber: event_count for runnumber, event_count in num_events} + for i in range(len(rows)): + runnum, nohitch, runtime, bco_stdev, bco_peak = rows[i] + event_count = event_count_dict.get(runnum, 1) + hist_nohitch.Fill(nohitch, event_count) + hist_nohitch_zoom.Fill(nohitch, event_count) + hist_runtime.Fill(runtime, event_count) + hist_bco_stdev.Fill(bco_stdev, event_count) + hist_bco_peak.Fill(bco_peak, event_count) + print(runnum, nohitch, runtime, bco_stdev, bco_peak, event_count) + +def save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree): + canvas0, canvas00, canvas1, canvas2, canvas3 = create_canvas() + canvas0.cd() + hist_nohitch.Draw() + hist_nohitch.Write() + canvas00.cd() + hist_nohitch_zoom.Draw() + hist_nohitch_zoom.Write() + canvas1.cd() + hist_runtime.Draw() + hist_runtime.Write() + canvas2.cd() + hist_bco_stdev.Draw() + hist_bco_stdev.Write() + canvas3.cd() + hist_bco_peak.Draw() + hist_bco_peak.Write() + tree.Write() + +def main(): + sfile = ROOT.TFile("SaveFile.root", "RECREATE") + rows = get_run_events() + hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak = create_histograms() + tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak = create_tree() + fill_histograms_and_tree(rows, hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree, _runnum, _runtime, _nohitch, _bco_stdev, _bco_peak) + save_histograms_and_tree(hist_nohitch, hist_nohitch_zoom, hist_runtime, hist_bco_stdev, hist_bco_peak, tree) + + # Extract runnumbers from rows + runnumbers = [row[0] for row in rows] + + # New histogram creation and filling with event count + num_events = get_num_events_from_daq(runnumbers) + hist_nohitch_event_count, hist_nohitch_zoom_event_count, hist_runtime_event_count, hist_bco_stdev_event_count, hist_bco_peak_event_count = create_histograms_with_event_count() + fill_histo_with_numevents(rows, num_events, hist_nohitch_event_count, hist_nohitch_zoom_event_count, hist_runtime_event_count, hist_bco_stdev_event_count, hist_bco_peak_event_count) + + # Save new histograms + hist_nohitch_event_count.Write() + hist_nohitch_zoom_event_count.Write() + hist_runtime_event_count.Write() + hist_bco_stdev_event_count.Write() + hist_bco_peak_event_count.Write() + sfile.Close() + +if __name__ == "__main__": + main() diff --git a/general_codes/Jaein/QA_database/put_in_database.py b/general_codes/Jaein/QA_database/put_in_database.py new file mode 100644 index 0000000..fae4d61 --- /dev/null +++ b/general_codes/Jaein/QA_database/put_in_database.py @@ -0,0 +1,153 @@ +import ROOT +import numpy as np +import glob +import os +import pyodbc +import csv +import collections +import math +from datetime import datetime + +ROOT.gROOT.SetBatch(True) + +def get_run_events(): + """Query database for run events""" + connection_string = ( + 'DRIVER={PostgreSQL};' + 'SERVER=sphnxdaqdbreplica;' + 'DATABASE=daq;' + 'PORT=5432;' + ) + with pyodbc.connect(connection_string) as conn: + cursor = conn.cursor() + cursor.execute("select * from run where runtype = 'physics' order by runnumber;") + return cursor.fetchall() + +def insert_data(runnum, dead_count, runtime, bco_stdev, bco_peak): + """Insert data into the database""" + connection_string = ( + 'DRIVER={PostgreSQL};' + 'SERVER=sphnxdbmaster;' + 'DATABASE=intt;' + 'PORT=5432;' + ) + with pyodbc.connect(connection_string) as conn: + cursor = conn.cursor() + cursor.execute( + "INSERT INTO intt_qa_expert (runnumber, nohitch, runtime, bco_stdev, bco_peak) VALUES (?, ?, ?, ?, ?);", + (runnum, dead_count, runtime, bco_stdev, bco_peak) + ) + conn.commit() + +def process_run_file(hot_file_path): + Dead = 0 + Cold = 0 + Hot = 0 + if os.path.isfile(hot_file_path): + file_hot = ROOT.TFile.Open(hot_file_path) + tree_hot = file_hot.Get("Multiple") + if tree_hot: + nEvent = tree_hot.GetEntries() + i = 0 + while i < nEvent: + tree_hot.GetEvent(i) + if tree_hot.Iflag == 1: + Dead += 1 + elif tree_hot.Iflag == 4: + Cold += 1 + elif tree_hot.Iflag == 8: + Hot += 1 + i += 1 + if not tree_hot: + return None + if not os.path.isfile(hot_file_path): + return None + return [Dead, Cold, Hot] + +def process_bco_file(bco_file_path): + BCO_stdev = 0 + BCO_peak = 0 + counter = [] + if os.path.isfile(bco_file_path): + file_bco = ROOT.TFile.Open(bco_file_path) + tree_bco_multi = file_bco.Get("Multiple") + tree_bco_single = file_bco.Get("Single") + if tree_bco_multi: + nEvent = tree_bco_multi.GetEntries() + i = 0 + while i < nEvent: + tree_bco_multi.GetEvent(i) + counter.append(tree_bco_multi.Ibco_diff) + i += 1 + c = collections.Counter(counter) + BCO_peak = c.most_common()[0][0] + if BCO_peak == -1: + if len(c.most_common()) > 1: + BCO_peak = c.most_common()[1][0] + if not tree_bco_multi: + return None + + if tree_bco_single: + nEvent = tree_bco_single.GetEntries() + i = 0 + while i < nEvent: + tree_bco_single.GetEvent(i) + BCO_stdev = tree_bco_single.DStdDev + if math.isnan(BCO_stdev): + BCO_stdev = -1 + i += 1 + if not tree_bco_single: + return None + if not os.path.isfile(bco_file_path): + return None + return [BCO_stdev, BCO_peak] + +def calculate_runtime(brtimestamp, ertimestamp): + """Calculate runtime in minutes""" + if not brtimestamp or not ertimestamp: + return -1 + try: + if isinstance(brtimestamp, datetime) and isinstance(ertimestamp, datetime): + start_time = brtimestamp + end_time = ertimestamp + else: + start_time = datetime.strptime(brtimestamp, '%Y-%m-%d %H:%M:%S') + end_time = datetime.strptime(ertimestamp, '%Y-%m-%d %H:%M:%S') + + runtime = (end_time - start_time).total_seconds() / 60 + return int(runtime) + except ValueError: + return -1 + + +def main(): + rows = get_run_events() + rownum = len(rows) + i = 6 + while i < rownum: + runnum = str(rows[i][0]) + brtimestamp = rows[i][2] + ertimestamp = rows[i][3] + runtime = calculate_runtime(brtimestamp, ertimestamp) + file_paths = f'/sphenix/user/jaein213/macros/inttcalib/test/hotmap_cdb_1106/hotmap_run_{runnum.zfill(8)}.root' + bco_file_paths = f'/sphenix/user/jaein213/macros/inttcalib/test/bcomap_cdb_1106/bcomap_run_{runnum.zfill(8)}.root' + if file_paths and bco_file_paths: + result = process_run_file(file_paths) + bco_result = process_bco_file(bco_file_paths) + if result and bco_result: + Dead, Cold, Hot = result + BCO_stdev, BCO_peak = bco_result + BCO_stdev = round(BCO_stdev, 4) + insert_data(runnum, Dead, runtime, BCO_stdev, BCO_peak) + print(runnum) + print(f"Dead: {Dead}, Cold: {Cold}, Hot: {Hot}, Runtime: {runtime} minutes, BCO_stdev: {BCO_stdev}, BCO_peak: {BCO_peak}") + else: + print(f"run {runnum} is missing") + else: + print(f"run {runnum} is missing") + i += 1 + # if(i==10): + # break + +if __name__ == "__main__": + main()