From ef445ffee54ec254298b8f58bc2399c5fd0e1320 Mon Sep 17 00:00:00 2001 From: ZhixiaoSu Date: Wed, 31 Jul 2024 11:15:41 -0700 Subject: [PATCH 1/4] No harp line warning. --- src/aind_ephys_rig_qc/temporal_alignment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aind_ephys_rig_qc/temporal_alignment.py b/src/aind_ephys_rig_qc/temporal_alignment.py index 4554c7b..5f85be3 100644 --- a/src/aind_ephys_rig_qc/temporal_alignment.py +++ b/src/aind_ephys_rig_qc/temporal_alignment.py @@ -214,7 +214,10 @@ def search_harp_line(recording, directory, pdf=None): # pick the line with even distribution overtime # and has short inter-event interval candidate_lines = lines_to_scan[(p_short > 0.5) & (p_value > 0.95)] - plt.suptitle(f"Harp line(s) {candidate_lines}") + if len(candidate_lines)>0: + plt.suptitle(f"Harp line(s) {candidate_lines}") + else: + plt.suptitle('Harp line not detected!', color='red') if pdf is not None: pdf.add_page() From be24ed25ca28412d36c528c148a49e147a0bbbc0 Mon Sep 17 00:00:00 2001 From: ZhixiaoSu Date: Wed, 31 Jul 2024 11:31:44 -0700 Subject: [PATCH 2/4] Append parameters to output record --- src/aind_ephys_rig_qc/generate_report.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/aind_ephys_rig_qc/generate_report.py b/src/aind_ephys_rig_qc/generate_report.py index b71807b..5eb00a6 100644 --- a/src/aind_ephys_rig_qc/generate_report.py +++ b/src/aind_ephys_rig_qc/generate_report.py @@ -11,6 +11,8 @@ import numpy as np import pandas as pd from open_ephys.analysis import Session +import shutil +import os from aind_ephys_rig_qc import __version__ as package_version from aind_ephys_rig_qc.pdf_utils import PdfReport @@ -321,6 +323,8 @@ def create_qc_plots( if __name__ == "__main__": + output_stream = io.StringIO() + sys.stdout = output_stream if len(sys.argv) != 3: print("Two input arguments are required:") print(" 1. A data directory") @@ -328,7 +332,6 @@ def create_qc_plots( else: with open(sys.argv[2], "r",) as f: parameters = json.load(f) - directory = sys.argv[1] print("Running generate_report.py with parameters:") @@ -338,4 +341,13 @@ def create_qc_plots( if not os.path.exists(directory): raise ValueError(f"Data directory {directory} does not exist.") + output_content = output_stream.getvalue() + + outfile = os.path.join(directory, "ephys-rig-QC_output.txt") + + with open(outfile, "a") as output_file: + output_file.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n") + output_file.write(output_content) + + generate_qc_report(directory, **parameters) From 6943be16deca62adccd050604bbb56767f4c3cd6 Mon Sep 17 00:00:00 2001 From: ZhixiaoSu Date: Wed, 31 Jul 2024 11:31:44 -0700 Subject: [PATCH 3/4] Append parameters to output record --- src/aind_ephys_rig_qc/generate_report.py | 24 +++++++++++++++++---- src/aind_ephys_rig_qc/qc_figures.py | 10 +++++++-- src/aind_ephys_rig_qc/temporal_alignment.py | 12 +++++------ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/aind_ephys_rig_qc/generate_report.py b/src/aind_ephys_rig_qc/generate_report.py index b71807b..9fadd63 100644 --- a/src/aind_ephys_rig_qc/generate_report.py +++ b/src/aind_ephys_rig_qc/generate_report.py @@ -94,7 +94,8 @@ def generate_qc_report( # optionally align to Harp timestamps print("Aligning timestamps to Harp clock...") align_timestamps_harp( - directory, pdf=pdf, + directory, + pdf=pdf, ) print("Creating QC plots...") @@ -275,7 +276,8 @@ def create_qc_plots( pdf.write(h=10, text=f"Duration: {duration} s") pdf.set_y(65) pdf.write( - h=10, text=f"Sample Rate: " f"{sample_rate} Hz", + h=10, + text=f"Sample Rate: " f"{sample_rate} Hz", ) pdf.set_y(70) pdf.write(h=10, text=f"Channels: {stream.samples.shape[1]}") @@ -321,14 +323,18 @@ def create_qc_plots( if __name__ == "__main__": + output_stream = io.StringIO() + sys.stdout = output_stream if len(sys.argv) != 3: print("Two input arguments are required:") print(" 1. A data directory") print(" 2. A JSON parameters file") else: - with open(sys.argv[2], "r",) as f: + with open( + sys.argv[2], + "r", + ) as f: parameters = json.load(f) - directory = sys.argv[1] print("Running generate_report.py with parameters:") @@ -338,4 +344,14 @@ def create_qc_plots( if not os.path.exists(directory): raise ValueError(f"Data directory {directory} does not exist.") + output_content = output_stream.getvalue() + + outfile = os.path.join(directory, "ephys-rig-QC_output.txt") + + with open(outfile, "a") as output_file: + output_file.write( + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n" + ) + output_file.write(output_content) + generate_qc_report(directory, **parameters) diff --git a/src/aind_ephys_rig_qc/qc_figures.py b/src/aind_ephys_rig_qc/qc_figures.py index 8e9bbe0..cecfb39 100644 --- a/src/aind_ephys_rig_qc/qc_figures.py +++ b/src/aind_ephys_rig_qc/qc_figures.py @@ -77,7 +77,11 @@ def plot_raw_data( def plot_power_spectrum( - data, start_frames, stream_name, sample_rate, chunk_size=10000, + data, + start_frames, + stream_name, + sample_rate, + chunk_size=10000, ): """ Plot the power spectrum of the data @@ -250,7 +254,9 @@ def plot_drift(directory, stream_name, block_index=0): ylim = [np.min(y_locs), np.max(y_locs)] fig = Figure(figsize=visualization_drift_params["figsize"]) - axs_drift = fig.subplots(ncols=recording.get_num_segments(),) + axs_drift = fig.subplots( + ncols=recording.get_num_segments(), + ) # for testing purposes if recording.get_total_duration() < 3: visualization_drift_params["n_skip"] = 1 diff --git a/src/aind_ephys_rig_qc/temporal_alignment.py b/src/aind_ephys_rig_qc/temporal_alignment.py index 5f85be3..2803422 100644 --- a/src/aind_ephys_rig_qc/temporal_alignment.py +++ b/src/aind_ephys_rig_qc/temporal_alignment.py @@ -214,10 +214,10 @@ def search_harp_line(recording, directory, pdf=None): # pick the line with even distribution overtime # and has short inter-event interval candidate_lines = lines_to_scan[(p_short > 0.5) & (p_value > 0.95)] - if len(candidate_lines)>0: + if len(candidate_lines) > 0: plt.suptitle(f"Harp line(s) {candidate_lines}") else: - plt.suptitle('Harp line not detected!', color='red') + plt.suptitle("Harp line not detected!", color="red") if pdf is not None: pdf.add_page() @@ -315,10 +315,10 @@ def align_timestamps( # noqa print("Processing stream: ", main_stream_name) main_stream_source_node_id = main_stream.metadata["source_node_id"] main_stream_sample_rate = main_stream.metadata["sample_rate"] - if 'PXIe' in main_stream_name and flip_NIDAQ: + if "PXIe" in main_stream_name and flip_NIDAQ: # flip the NIDAQ stream if sync line is inverted between NIDAQ # and main stream - print('Flipping NIDAQ stream as main stream...') + print("Flipping NIDAQ stream as main stream...") main_stream_events = events[ (events.stream_name == main_stream_name) & (events.processor_id == main_stream_source_node_id) @@ -470,8 +470,8 @@ def align_timestamps( # noqa print("Processing stream: ", stream_name) source_node_id = stream.metadata["source_node_id"] sample_rate = stream.metadata["sample_rate"] - if 'PXIe' in stream_name and flip_NIDAQ: - print('Flipping NIDAQ stream...') + if "PXIe" in stream_name and flip_NIDAQ: + print("Flipping NIDAQ stream...") # flip the NIDAQ stream if sync line is inverted # between NIDAQ and main stream events_for_stream = events[ From 11510e4cb1747e67b253a31bf4f4a4f7dd3ecd1e Mon Sep 17 00:00:00 2001 From: ZhixiaoSu Date: Wed, 31 Jul 2024 12:01:05 -0700 Subject: [PATCH 4/4] Package version control --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 13a6128..08a9b87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,8 @@ dependencies = [ 'scipy', 'rich', 'spikeinterface[full]==0.100.7', - 'harp-python @ git+https://github.com/jsiegle/harp-python@decode-clock' + 'harp-python @ git+https://github.com/jsiegle/harp-python@decode-clock', + 'numpy==1.26.4' ] [project.optional-dependencies]