diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index be326e25..1630dafa 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -18,5 +18,6 @@ jobs: docker run -t iqtlabs/gamutrf:latest gamutrf-api --help docker run -t iqtlabs/gamutrf:latest gamutrf-samples2raw --help docker run -t iqtlabs/gamutrf:latest gamutrf-freqxlator --help + docker run -t iqtlabs/gamutrf:latest gamutrf-waterfall --help sudo apt-get update && sudo apt-get install -qy python3-pip docker compose -f orchestrator.yml -f worker.yml -f monitoring.yml -f specgram.yml build diff --git a/gamutrf/__main__.py b/gamutrf/__main__.py index de10c76a..74a4c7a4 100644 --- a/gamutrf/__main__.py +++ b/gamutrf/__main__.py @@ -5,6 +5,7 @@ from gamutrf.scan import main as scan_main from gamutrf.sigfinder import main as sigfinder_main from gamutrf.specgram import main as specgram_main +from gamutrf.waterfall import main as waterfall_main def api(): @@ -35,3 +36,8 @@ def sigfinder(): def specgram(): """Entrypoint for specgram""" specgram_main() + + +def waterfall(): + """Entrypoint for waterfall""" + waterfall_main() diff --git a/gamutrf/flask_handler.py b/gamutrf/flask_handler.py index 6cabc865..d828d548 100644 --- a/gamutrf/flask_handler.py +++ b/gamutrf/flask_handler.py @@ -13,7 +13,9 @@ def __init__(self, options, check_options, banned_args): self.app.add_url_rule("/reconf", "reconf", self.reconf) self.request = request self.thread = threading.Thread( - target=self.app.run, kwargs={"port": options.apiport}, daemon=True + target=self.app.run, + kwargs={"host": "0.0.0.0", "port": options.apiport}, # nosec + daemon=True, ) def start(self): diff --git a/gamutrf/waterfall.py b/gamutrf/waterfall.py index d5cc36d4..51349f8f 100644 --- a/gamutrf/waterfall.py +++ b/gamutrf/waterfall.py @@ -3,8 +3,11 @@ import datetime import json import logging +import os import signal import sys +import tempfile +import threading import time import warnings from pathlib import Path @@ -12,6 +15,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np +from flask import Flask, current_app from matplotlib.collections import LineCollection from matplotlib.ticker import MultipleLocator, AutoMinorLocator from scipy.ndimage import gaussian_filter @@ -19,13 +23,21 @@ from gamutrf.zmqreceiver import ZmqReceiver, parse_scanners -matplotlib.use("GTK3Agg") warnings.filterwarnings(action="ignore", message="Mean of empty slice") warnings.filterwarnings(action="ignore", message="All-NaN slice encountered") warnings.filterwarnings(action="ignore", message="Degrees of freedom <= 0 for slice.") -def draw_waterfall(mesh, fig, ax, data, cmap): +def safe_savefig(path): + basename = os.path.basename(path) + dirname = os.path.dirname(path) + tmp_path = os.path.join(dirname, "." + basename) + plt.savefig(tmp_path) + os.rename(tmp_path, path) + logging.info("wrote %s", path) + + +def draw_waterfall(mesh, ax, data, cmap): mesh.set_array(cmap(data)) ax.draw_artist(mesh) @@ -55,6 +67,107 @@ def filter_peaks(peaks, properties): return peaks, properties +def save_detections( + save_path, + scan_time, + min_freq, + max_freq, + scan_configs, + previous_scan_config, + peaks, + properties, + psd_x_edges, + detection_type, +): + detection_config_save_path = str( + Path( + save_path, + "detections", + f"detections_scan_config_{scan_time}.json", + ) + ) + detection_save_path = str( + Path( + save_path, + "detections", + f"detections_{scan_time}.csv", + ) + ) + + if previous_scan_config is None or previous_scan_config != scan_configs: + previous_scan_config = scan_configs + with open(detection_config_save_path, "w", encoding="utf8") as f: + json.dump( + { + "timestamp": scan_time, + "min_freq": min_freq, + "max_freq": max_freq, + "scan_configs": scan_configs, + }, + f, + indent=4, + ) + with open(detection_save_path, "w", encoding="utf8") as detection_csv: + writer = csv.writer(detection_csv) + writer.writerow( + [ + "timestamp", + "start_freq", + "end_freq", + "dB", + "type", + ] + ) + + with open(detection_save_path, "a", encoding="utf8") as detection_csv: + writer = csv.writer(detection_csv) + for i in range(len(peaks)): + writer.writerow( + [ + scan_time, # timestamp + psd_x_edges[properties["left_ips"][i].astype(int)], # start_freq + psd_x_edges[properties["right_ips"][i].astype(int)], # end_freq + properties["peak_heights"][i], # dB + detection_type, # type + ] + ) + return previous_scan_config + + +def save_waterfall( + save_path, + save_time, + last_save_time, + scan_time, + scan_times, + scan_config_history, +): + now = datetime.datetime.now() + if last_save_time is None: + last_save_time = now + + if now - last_save_time > datetime.timedelta(minutes=save_time): + waterfall_save_path = str( + Path(save_path, "waterfall", f"waterfall_{scan_time}.png") + ) + safe_savefig(waterfall_save_path) + + save_scan_configs = { + "start_scan_timestamp": scan_times[0], + "start_scan_config": scan_config_history[scan_times[0]], + "end_scan_timestamp": scan_times[-1], + "end_scan_config": scan_config_history[scan_times[-1]], + } + config_save_path = str(Path(save_path, "waterfall", f"config_{scan_time}.json")) + with open(config_save_path, "w", encoding="utf8") as f: + json.dump(save_scan_configs, f, indent=4) + + last_save_time = now + logging.info(f"Saving {waterfall_save_path}") + + return last_save_time + + def argument_parser(): parser = argparse.ArgumentParser(description="waterfall plotter from scan data") parser.add_argument( @@ -94,9 +207,182 @@ def argument_parser(): type=str, help="Scanner endpoints to use.", ) + parser.add_argument( + "--port", + default=0, + type=int, + help="If set, serve waterfall on this port.", + ) return parser +def reset_fig( + savefig_path, + fig, + min_freq, + max_freq, + freq_resolution, + cmap, + db_min, + db_max, + psd_db_resolution, + snr_min, + snr_max, + plot_snr, + minor_tick_separator, + major_tick_separator, + marker_distance, + top_n, + freq_data, + db_data, + X, + Y, +): + # RESET FIGURE + fig.clf() + plt.tight_layout() + plt.subplots_adjust(hspace=0.15) + ax_psd = fig.add_subplot(3, 1, 1) + ax = fig.add_subplot(3, 1, (2, 3)) + psd_title = ax_psd.text( + 0.5, 1.05, "", transform=ax_psd.transAxes, va="center", ha="center" + ) + + # PSD + XX, YY = np.meshgrid( + np.linspace( + min_freq, + max_freq, + int((max_freq - min_freq) / (freq_resolution) + 1), + ), + np.linspace(db_min, db_max, psd_db_resolution), + ) + psd_x_edges = XX[0] + psd_y_edges = YY[:, 0] + + _mesh_psd = ax_psd.pcolormesh(XX, YY, np.zeros(XX[:-1, :-1].shape), shading="flat") + (peak_lns,) = ax_psd.plot( + X[0], + db_min * np.ones(freq_data.shape[1]), + color="white", + marker="^", + markersize=12, + linestyle="none", + fillstyle="full", + ) + (max_psd_ln,) = ax_psd.plot( + X[0], + db_min * np.ones(freq_data.shape[1]), + color="red", + marker=",", + linestyle=":", + markevery=marker_distance, + label="max", + ) + (min_psd_ln,) = ax_psd.plot( + X[0], + db_min * np.ones(freq_data.shape[1]), + color="pink", + marker=",", + linestyle=":", + markevery=marker_distance, + label="min", + ) + (mean_psd_ln,) = ax_psd.plot( + X[0], + db_min * np.ones(freq_data.shape[1]), + color="cyan", + marker="^", + markersize=8, + fillstyle="none", + linestyle=":", + markevery=marker_distance, + label="mean", + ) + (current_psd_ln,) = ax_psd.plot( + X[0], + db_min * np.ones(freq_data.shape[1]), + color="red", + marker="o", + markersize=8, + fillstyle="none", + linestyle=":", + markevery=marker_distance, + label="current", + ) + ax_psd.legend(loc="center left", bbox_to_anchor=(1, 0.5)) + ax_psd.set_ylabel("dB") + + # SPECTROGRAM + mesh = ax.pcolormesh(X, Y, db_data, shading="nearest") + top_n_lns = [] + for _ in range(top_n): + (ln,) = ax.plot( + [X[0][0]] * len(Y[:, 0]), + Y[:, 0], + color="brown", + linestyle=":", + alpha=0, + ) + top_n_lns.append(ln) + + ax.set_xlabel("MHz") + ax.set_ylabel("Time") + + # COLORBAR + sm = plt.cm.ScalarMappable(cmap=cmap) + sm.set_clim(vmin=db_min, vmax=db_max) + + if plot_snr: + sm.set_clim(vmin=snr_min, vmax=snr_max) + cbar_ax = fig.add_axes([0.92, 0.10, 0.03, 0.5]) + cbar = fig.colorbar(sm, cax=cbar_ax) + cbar.set_label("dB", rotation=0) + + # SPECTROGRAM TITLE + _title = ax.text(0.5, 1.05, "", transform=ax.transAxes, va="center", ha="center") + + ax.xaxis.set_major_locator(MultipleLocator(major_tick_separator)) + ax.xaxis.set_major_formatter("{x:.0f}") + ax.xaxis.set_minor_locator(minor_tick_separator) + ax_psd.xaxis.set_major_locator(MultipleLocator(major_tick_separator)) + ax_psd.xaxis.set_major_formatter("{x:.0f}") + ax_psd.xaxis.set_minor_locator(minor_tick_separator) + + ax_psd.yaxis.set_animated(True) + cbar_ax.yaxis.set_animated(True) + ax.yaxis.set_animated(True) + plt.show(block=False) + plt.pause(0.1) + + background = fig.canvas.copy_from_bbox(fig.bbox) + + ax.draw_artist(mesh) + fig.canvas.blit(ax.bbox) + if savefig_path: + safe_savefig(savefig_path) + + for ln in top_n_lns: + ln.set_alpha(0.75) + return ( + ax, + ax_psd, + min_psd_ln, + current_psd_ln, + mean_psd_ln, + max_psd_ln, + peak_lns, + psd_title, + psd_x_edges, + psd_y_edges, + cbar, + cbar_ax, + sm, + mesh, + background, + ) + + def waterfall( min_freq, max_freq, @@ -108,7 +394,11 @@ def waterfall( save_time, detection_type, scanners, + engine, + savefig_path, ): + matplotlib.use(engine) + # OTHER PARAMETERS cmap = plt.get_cmap("viridis") cmap_psd = plt.get_cmap("turbo") @@ -118,7 +408,6 @@ def waterfall( snr_max = 50 waterfall_height = 100 # number of waterfall rows scale = 1e6 - zmq_sleep_time = 1 freq_resolution = sampling_rate / fft_len draw_rate = 1 @@ -136,8 +425,6 @@ def waterfall( counter = 0 y_ticks = [] y_labels = [] - psd_x_edges = None - psd_y_edges = None background = None top_n_lns = [] last_save_time = None @@ -148,8 +435,6 @@ def waterfall( hl = None detection_text = [] previous_scan_config = None - detection_config_save_path = None - detection_save_path = None plt.rcParams["savefig.facecolor"] = "#2A3459" plt.rcParams["figure.facecolor"] = "#2A3459" @@ -211,146 +496,53 @@ def sig_handler(_sig=None, _frame=None): freq_data = np.empty(X.shape) freq_data.fill(np.nan) - def onresize(event): + def onresize(_event): global init_fig init_fig = True fig.canvas.mpl_connect("resize_event", onresize) while True: - if init_fig: - # RESET FIGURE - fig.clf() - plt.tight_layout() - plt.subplots_adjust(hspace=0.15) - ax_psd = fig.add_subplot(3, 1, 1) - ax = fig.add_subplot(3, 1, (2, 3)) - psd_title = ax_psd.text( - 0.5, 1.05, "", transform=ax_psd.transAxes, va="center", ha="center" - ) - - # PSD - XX, YY = np.meshgrid( - np.linspace( - min_freq, - max_freq, - int((max_freq - min_freq) / (freq_resolution) + 1), - ), - np.linspace(db_min, db_max, psd_db_resolution), - ) - psd_x_edges = XX[0] - psd_y_edges = YY[:, 0] - - mesh_psd = ax_psd.pcolormesh( - XX, YY, np.zeros(XX[:-1, :-1].shape), shading="flat" - ) - (peak_lns,) = ax_psd.plot( - X[0], - db_min * np.ones(freq_data.shape[1]), - color="white", - marker="^", - markersize=12, - linestyle="none", - fillstyle="full", - ) - (max_psd_ln,) = ax_psd.plot( - X[0], - db_min * np.ones(freq_data.shape[1]), - color="red", - marker=",", - linestyle=":", - markevery=marker_distance, - label="max", - ) - (min_psd_ln,) = ax_psd.plot( - X[0], - db_min * np.ones(freq_data.shape[1]), - color="pink", - marker=",", - linestyle=":", - markevery=marker_distance, - label="min", - ) - (mean_psd_ln,) = ax_psd.plot( - X[0], - db_min * np.ones(freq_data.shape[1]), - color="cyan", - marker="^", - markersize=8, - fillstyle="none", - linestyle=":", - markevery=marker_distance, - label="mean", - ) - (current_psd_ln,) = ax_psd.plot( - X[0], - db_min * np.ones(freq_data.shape[1]), - color="red", - marker="o", - markersize=8, - fillstyle="none", - linestyle=":", - markevery=marker_distance, - label="current", - ) - ax_psd.legend(loc="center left", bbox_to_anchor=(1, 0.5)) - ax_psd.set_ylabel("dB") - - # SPECTROGRAM - mesh = ax.pcolormesh(X, Y, db_data, shading="nearest") - top_n_lns = [] - for _ in range(top_n): - (ln,) = ax.plot( - [X[0][0]] * len(Y[:, 0]), - Y[:, 0], - color="brown", - linestyle=":", - alpha=0, - ) - top_n_lns.append(ln) - - ax.set_xlabel("MHz") - ax.set_ylabel("Time") - - # COLORBAR - sm = plt.cm.ScalarMappable(cmap=cmap) - sm.set_clim(vmin=db_min, vmax=db_max) - - if plot_snr: - sm.set_clim(vmin=snr_min, vmax=snr_max) - cbar_ax = fig.add_axes([0.92, 0.10, 0.03, 0.5]) - cbar = fig.colorbar(sm, cax=cbar_ax) - cbar.set_label("dB", rotation=0) - - # SPECTROGRAM TITLE - title = ax.text( - 0.5, 1.05, "", transform=ax.transAxes, va="center", ha="center" - ) - - ax.xaxis.set_major_locator(MultipleLocator(major_tick_separator)) - ax.xaxis.set_major_formatter("{x:.0f}") - ax.xaxis.set_minor_locator(minor_tick_separator) - ax_psd.xaxis.set_major_locator(MultipleLocator(major_tick_separator)) - ax_psd.xaxis.set_major_formatter("{x:.0f}") - ax_psd.xaxis.set_minor_locator(minor_tick_separator) - - ax_psd.yaxis.set_animated(True) - cbar_ax.yaxis.set_animated(True) - ax.yaxis.set_animated(True) - plt.show(block=False) - plt.pause(0.1) - - background = fig.canvas.copy_from_bbox(fig.bbox) - - ax.draw_artist(mesh) - fig.canvas.blit(ax.bbox) - - for ln in top_n_lns: - ln.set_alpha(0.75) - - init_fig = False - - else: + ( + ax, + ax_psd, + min_psd_ln, + current_psd_ln, + mean_psd_ln, + max_psd_ln, + peak_lns, + psd_title, + psd_x_edges, + psd_y_edges, + cbar, + cbar_ax, + sm, + mesh, + background, + ) = reset_fig( + savefig_path, + fig, + min_freq, + max_freq, + freq_resolution, + cmap, + db_min, + db_max, + psd_db_resolution, + snr_min, + snr_max, + plot_snr, + minor_tick_separator, + major_tick_separator, + marker_distance, + top_n, + freq_data, + db_data, + X, + Y, + ) + init_fig = False + while not init_fig: time.sleep(0.1) results = [] while True: @@ -388,7 +580,7 @@ def onresize(event): db_data[-1, :] = np.nan db_data[-1][idx] = db - data, xedge, yedge = np.histogram2d( + data, _xedge, _yedge = np.histogram2d( freq_data[~np.isnan(freq_data)].flatten(), db_data[~np.isnan(db_data)].flatten(), density=False, @@ -509,64 +701,18 @@ def onresize(event): peaks, properties = filter_peaks(peaks, properties) if save_path: - if ( - previous_scan_config is None - or previous_scan_config != scan_configs - ): - previous_scan_config = scan_configs - detection_config_save_path = str( - Path( - save_path, - "detections", - f"detections_scan_config_{scan_time}.json", - ) - ) - with open(detection_config_save_path, "w") as f: - json.dump( - { - "timestamp": scan_time, - "min_freq": min_freq, - "max_freq": max_freq, - "scan_configs": scan_configs, - }, - f, - indent=4, - ) - detection_save_path = str( - Path( - save_path, - "detections", - f"detections_{scan_time}.csv", - ) - ) - with open(detection_save_path, "w") as detection_csv: - writer = csv.writer(detection_csv) - writer.writerow( - [ - "timestamp", - "start_freq", - "end_freq", - "dB", - "type", - ] - ) - - with open(detection_save_path, "a") as detection_csv: - writer = csv.writer(detection_csv) - for i in range(len(peaks)): - writer.writerow( - [ - scan_time, # timestamp - psd_x_edges[ - properties["left_ips"][i].astype(int) - ], # start_freq - psd_x_edges[ - properties["right_ips"][i].astype(int) - ], # end_freq - properties["peak_heights"][i], # dB - detection_type, # type - ] - ) + previous_scan_config = save_detections( + save_path, + scan_time, + min_freq, + max_freq, + scan_configs, + previous_scan_config, + peaks, + properties, + psd_x_edges, + detection_type, + ) peak_lns.set_xdata(psd_x_edges[peaks]) peak_lns.set_ydata(properties["width_heights"]) @@ -652,7 +798,7 @@ def onresize(event): ax_psd.draw_artist(mean_psd_ln) ax_psd.draw_artist(current_psd_ln) - draw_waterfall(mesh, fig, ax, db_norm, cmap) + draw_waterfall(mesh, ax, db_norm, cmap) draw_title(ax_psd, psd_title, title_text) sm.set_clim(vmin=db_min, vmax=db_max) @@ -672,38 +818,50 @@ def onresize(event): fig.canvas.blit(cbar_ax.bbox) fig.canvas.blit(fig.bbox) fig.canvas.flush_events() + if savefig_path: + safe_savefig(savefig_path) logging.info(f"Plotting {row_time}") if save_path: - if last_save_time is None: - last_save_time = datetime.datetime.now() - - if ( - datetime.datetime.now() - last_save_time - > datetime.timedelta(minutes=save_time) - ): - waterfall_save_path = str( - Path( - save_path, "waterfall", f"waterfall_{scan_time}.png" - ) - ) - fig.savefig(waterfall_save_path) - - save_scan_configs = { - "start_scan_timestamp": scan_times[0], - "start_scan_config": scan_config_history[scan_times[0]], - "end_scan_timestamp": scan_times[-1], - "end_scan_config": scan_config_history[scan_times[-1]], - } - config_save_path = str( - Path(save_path, "waterfall", f"config_{scan_time}.json") - ) - with open(config_save_path, "w") as f: - json.dump(save_scan_configs, f, indent=4) + last_save_time = save_waterfall( + save_path, + save_time, + last_save_time, + scan_time, + scan_times, + scan_config_history, + ) + - last_save_time = datetime.datetime.now() - logging.info(f"Saving {waterfall_save_path}") +class FlaskHandler: + def __init__(self, savefig_path, port, refresh=5): + self.savefig_path = savefig_path + self.refresh = refresh + self.app = Flask(__name__, static_folder=os.path.dirname(savefig_path)) + self.savefig_file = os.path.basename(self.savefig_path) + self.app.add_url_rule("/", "", self.serve) + self.app.add_url_rule( + "/" + self.savefig_file, self.savefig_file, self.serve_waterfall + ) + self.thread = threading.Thread( + target=self.app.run, + kwargs={"host": "0.0.0.0", "port": port}, # nosec + daemon=True, + ) + + def start(self): + self.thread.start() + + def serve(self): + return ( + '
' + % (self.refresh, self.savefig_file), + 200, + ) + + def serve_waterfall(self): + return current_app.send_static_file(self.savefig_file) def main(): @@ -711,14 +869,13 @@ def main(): parser = argument_parser() args = parser.parse_args() - save_path = args.save_path detection_type = args.detection_type - if save_path: - Path(save_path, "waterfall").mkdir(parents=True, exist_ok=True) + if args.save_path: + Path(args.save_path, "waterfall").mkdir(parents=True, exist_ok=True) if detection_type: - Path(save_path, "detections").mkdir(parents=True, exist_ok=True) + Path(args.save_path, "detections").mkdir(parents=True, exist_ok=True) detection_type = detection_type.lower() if detection_type in ["wb", "wide band", "wideband"]: @@ -727,18 +884,32 @@ def main(): detection_type = "narrowband" else: raise ValueError("detection_type must be 'narrowband' or 'wideband'") - waterfall( - args.min_freq, - args.max_freq, - args.plot_snr, - args.n_detect, - args.nfft, - args.sampling_rate, - save_path, - args.save_time, - detection_type, - args.scanners, - ) + + with tempfile.TemporaryDirectory() as tempdir: + flask = None + savefig_path = None + engine = "GTK3Agg" + + if args.port: + engine = "agg" + savefig_path = os.path.join(tempdir, "waterfall.png") + flask = FlaskHandler(savefig_path, args.port) + flask.start() + + waterfall( + args.min_freq, + args.max_freq, + args.plot_snr, + args.n_detect, + args.nfft, + args.sampling_rate, + args.save_path, + args.save_time, + detection_type, + args.scanners, + engine, + savefig_path, + ) if __name__ == "__main__": diff --git a/orchestrator.yml b/orchestrator.yml index be53d06a..5042913d 100644 --- a/orchestrator.yml +++ b/orchestrator.yml @@ -73,8 +73,6 @@ services: - gamutrf-sigfinder - --scanners=gamutrf:10000 - --log=/logs/scan.csv - - --fftlog=/logs/fft.csv - - --fftgraph=/logs/fft.png - --width=10 - --prominence=2 - --threshold=-25 @@ -83,6 +81,22 @@ services: environment: - "PEAK_TRIGGER=0" - "PIN_TRIGGER=17" + waterfall: + restart: always + image: iqtlabs/gamutrf:latest + networks: + - gamutrf + ports: + - '9003:9003' + volumes: + - '${VOL_PREFIX}:/logs' + command: + - gamutrf-waterfall + - --scanners=gamutrf:10000 + - --save_path=/logs + - --port=9003 + - --min_freq=100e6 + - --max_freq=1e9 watchtower: image: containrrr/watchtower:latest restart: always diff --git a/pyproject.toml b/pyproject.toml index d4fb3814..186a0040 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ gamutrf-scan = 'gamutrf.__main__:scan' gamutrf-sigfinder = 'gamutrf.__main__:sigfinder' gamutrf-specgram = 'gamutrf.__main__:specgram' +gamutrf-waterfall = 'gamutrf.__main__:waterfall' + [tool.poetry.urls] homepage = "https://github.com/IQTLabs/gamutRF"