From f7edbf8e5366a9a46f5c11d17f26b3a0d1f7ec7f Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Sun, 10 Dec 2023 20:35:35 +0000 Subject: [PATCH] use common get_samples(), remove multiple parse_filename. --- augment/augment.py | 52 +++--------------- gamutrf/grsource.py | 3 +- gamutrf/offline.py | 9 ++-- gamutrf/sample_reader.py | 87 ++++++++++++++---------------- gamutrf/samples2raw.py | 11 ++-- gamutrf/specgram.py | 21 +++----- gamutrf/utils.py | 36 +------------ gamutrf/waterfall_samples.py | 24 ++++----- tests/test_birdseye_rssi.py | 6 ++- tests/test_sample_reader.py | 66 ++++++++++------------- tests/test_sdr_recorder.py | 4 +- tests/test_specgram.py | 14 ++--- tests/test_utils.py | 101 +++++++++++++++++++++-------------- 13 files changed, 177 insertions(+), 257 deletions(-) diff --git a/augment/augment.py b/augment/augment.py index 37e1467b..cb2e7e16 100755 --- a/augment/augment.py +++ b/augment/augment.py @@ -13,62 +13,21 @@ uniform_discrete_distribution, ) from torchsig.utils.types import SignalData, SignalDescription -from gamutrf.sample_reader import read_recording, parse_filename +from gamutrf.sample_reader import read_recording, get_samples -def make_signal(samples, sample_rate, center_frequency): +def make_signal(samples, meta): num_iq_samples = samples.shape[0] desc = SignalDescription( - sample_rate=sample_rate, + sample_rate=meta["sample_rate"], num_iq_samples=num_iq_samples, - center_frequency=center_frequency, + center_frequency=meta["center_frequency"], ) # TODO: subclass SignalData with alternate constructor that can take just numpy array signal = SignalData(samples.tobytes(), np.float32, np.complex128, desc) return signal -def get_nosigmf_file(filename): - meta = parse_filename(filename) - sample_rate = meta["sample_rate"] - sample_dtype = meta["sample_dtype"] - sample_len = meta["sample_len"] - center_frequency = meta["freq_center"] - samples = None - for samples_buffer in read_recording( - filename, sample_rate, sample_dtype, sample_len, max_sample_secs=None - ): - if samples is None: - samples = samples_buffer - else: - samples = np.concatenate([samples, samples_buffer]) - signal = make_signal(samples, sample_rate, center_frequency) - return filename, signal - - -def get_signal(filename): - if not os.path.exists(filename): - raise FileNotFoundError(filename) - meta_ext = filename.find(".sigmf-meta") - if meta_ext == -1: - return get_nosigmf_file(filename) - - meta = sigmf.sigmffile.fromfile(filename) - data_filename = filename[:meta_ext] - meta.set_data_file(data_filename) - # read_samples() always converts to host cf32. - samples = meta.read_samples() - global_meta = meta.get_global_info() - sample_rate = global_meta["core:sample_rate"] - sample_type = global_meta["core:datatype"] - captures_meta = meta.get_captures() - center_frequency = None - if captures_meta: - center_frequency = captures_meta[0].get("core:frequency", None) - signal = make_signal(samples, sample_rate, center_frequency) - return data_filename, signal - - def write_signal(filename, signal, transforms_text): first_desc = signal.signal_description[0] signal.iq_data = signal.iq_data.astype(np.complex64) @@ -133,7 +92,8 @@ def argument_parser(): def main(): options = argument_parser().parse_args() - data_filename, signal = get_signal(options.filename) + data_filename, samples, meta = get_samples(options.filename) + signal = make_signal(samples, meta) augment(signal, data_filename, options.outdir, options.n, options.transforms) diff --git a/gamutrf/grsource.py b/gamutrf/grsource.py index 345af2a9..f1dd4eb2 100644 --- a/gamutrf/grsource.py +++ b/gamutrf/grsource.py @@ -50,7 +50,8 @@ def __init__( in_sig=None, out_sig=[np.complex64], ) - _, self.samples, self.center_freq = get_samples(input_file) + _, self.samples, meta = get_samples(input_file) + self.center_freq = meta["center_frequency"] self.n_samples = len(self.samples) self.i = 0 self.message_port_register_in(pmt.intern(cmd_port)) diff --git a/gamutrf/offline.py b/gamutrf/offline.py index 62dc80de..ae142d70 100644 --- a/gamutrf/offline.py +++ b/gamutrf/offline.py @@ -7,7 +7,7 @@ from gnuradio import iqtlabs from gamutrf.grscan import grscan -from gamutrf.sample_reader import parse_filename +from gamutrf.sample_reader import get_samples def argument_parser(): @@ -63,19 +63,20 @@ def main(): options = argument_parser().parse_args() filename = options.filename out_dir = os.path.dirname(filename) - meta = parse_filename(filename) + _data_filename, _samples, meta = get_samples(filename) + freq_start = int(meta["center_frequency"] - (meta["sample_rate"] / 2)) tb = grscan( db_clamp_floor=-1e9, fft_batch_size=256, freq_end=0, - freq_start=meta["freq_center"], + freq_start=freq_start, inference_min_db=-1e9, inference_output_dir=out_dir, iqtlabs=iqtlabs, n_image=options.n_image, nfft=options.nfft, pretune=True, - samp_rate=meta["sample_rate"], + samp_rate=int(meta["sample_rate"]), sample_dir=out_dir, sdr="file:" + filename, tune_step_fft=options.tune_step_fft, diff --git a/gamutrf/sample_reader.py b/gamutrf/sample_reader.py index 907ea3d7..3dad8a74 100644 --- a/gamutrf/sample_reader.py +++ b/gamutrf/sample_reader.py @@ -8,9 +8,7 @@ import numpy as np from gamutrf.utils import SAMPLE_DTYPES, SAMPLE_FILENAME_RE, is_fft -FFT_FILENAME_RE = re.compile( - r"^.+_([0-9]+)_([0-9]+)points_([0-9]+)Hz_([0-9]+)sps\.(s\d+|raw).*$" -) +POINTS_RE = re.compile(r"^.+\D([0-9]+)points_.+$") def get_reader(filename): @@ -36,51 +34,41 @@ def default_reader(x): def parse_filename(filename): + timestamp = None + nfft = None + center_frequency = None + sample_rate = None + sample_type = None + # FFT is always float not matter the original sample type. if is_fft(filename): sample_type = "raw" - match = FFT_FILENAME_RE.match(filename) - try: - timestamp = int(match.group(1)) - nfft = int(match.group(2)) - freq_center = int(match.group(3)) - sample_rate = int(match.group(4)) - # sample_type = match.group(3) - except AttributeError: - timestamp = None - nfft = None - freq_center = None - sample_rate = None - sample_type = None - else: - match = SAMPLE_FILENAME_RE.match(filename) - nfft = None - try: - timestamp = int(match.group(1)) - freq_center = int(match.group(2)) - sample_rate = int(match.group(3)) - sample_type = match.group(4) - except AttributeError: - timestamp = None - freq_center = None - sample_rate = None - sample_type = None + match = POINTS_RE.match(filename) + if match is not None: + nfft = int(match.group(1)) + + match = SAMPLE_FILENAME_RE.match(filename) + if match is not None: + timestamp, center_frequency, sample_rate, sample_type = ( + int(match.group(1)), + int(match.group(2)), + int(match.group(3)), + match.group(4), + ) sample_dtype, sample_type = SAMPLE_DTYPES.get(sample_type, (None, None)) sample_bits = None sample_len = None if sample_dtype: if is_fft(filename): - sample_dtype = np.float32 - sample_bits = 32 - sample_len = 4 + sample_dtype = np.dtype([("i", np.float32), ("q", np.float32)]) else: sample_dtype = np.dtype([("i", sample_dtype), ("q", sample_dtype)]) - sample_bits = sample_dtype[0].itemsize * 8 - sample_len = sample_dtype[0].itemsize * 2 - file_info = { + sample_bits = sample_dtype[0].itemsize * 8 + sample_len = sample_dtype[0].itemsize * 2 + meta = { "filename": filename, - "freq_center": freq_center, + "center_frequency": center_frequency, "sample_rate": sample_rate, "sample_dtype": sample_dtype, "sample_len": sample_len, @@ -89,7 +77,7 @@ def parse_filename(filename): "nfft": nfft, "timestamp": timestamp, } - return file_info + return meta def read_recording( @@ -143,19 +131,19 @@ def read_recording( def get_nosigmf_samples(filename): meta = parse_filename(filename) - sample_rate = meta["sample_rate"] - sample_dtype = meta["sample_dtype"] - sample_len = meta["sample_len"] - center_frequency = meta["freq_center"] samples = None for samples_buffer in read_recording( - filename, sample_rate, sample_dtype, sample_len, max_sample_secs=None + filename, + meta["sample_rate"], + meta["sample_dtype"], + meta["sample_len"], + max_sample_secs=None, ): if samples is None: samples = samples_buffer else: samples = np.concatenate([samples, samples_buffer]) - return filename, samples, center_frequency + return filename, samples, meta def get_samples(filename): @@ -168,13 +156,18 @@ def get_samples(filename): meta = sigmf.sigmffile.fromfile(filename) data_filename = filename[:meta_ext] meta.set_data_file(data_filename) - # read_samples() always converts to host cf32. samples = meta.read_samples() global_meta = meta.get_global_info() - sample_rate = global_meta["core:sample_rate"] - sample_type = global_meta["core:datatype"] captures_meta = meta.get_captures() center_frequency = None if captures_meta: center_frequency = captures_meta[0].get("core:frequency", None) - return data_filename, samples, center_frequency + meta = { + "sample_rate": global_meta["core:sample_rate"], + "sample_dtype": global_meta["core:datatype"], + "sample_len": samples[ + 0 + ].itemsize, # read_samples() always converts to host cf32. + "center_frequency": center_frequency, + } + return data_filename, samples, meta diff --git a/gamutrf/samples2raw.py b/gamutrf/samples2raw.py index 02ac04a9..ea9f0088 100644 --- a/gamutrf/samples2raw.py +++ b/gamutrf/samples2raw.py @@ -1,7 +1,7 @@ import argparse import subprocess -from gamutrf.utils import parse_filename +from gamutrf.sample_reader import parse_filename from gamutrf.utils import replace_ext @@ -16,10 +16,11 @@ def make_procs_args(sample_filename, outfmt): procs_args.append(["zstdcat", sample_filename]) out_filename = replace_ext(out_filename, "") - _, _, sample_rate, _sample_dtype, _sample_len, in_format, in_bits = parse_filename( - out_filename - ) - print(_, sample_rate, _sample_dtype, _sample_len, in_format, in_bits) + meta = parse_filename(out_filename) + sample_rate = meta["sample_rate"] + in_format = meta["sample_type"] + in_bits = meta["sample_bits"] + print(meta) out_filename = replace_ext(out_filename, "raw", all_ext=True) sox_in = sample_filename if procs_args: diff --git a/gamutrf/specgram.py b/gamutrf/specgram.py index a074d12e..8d47bafc 100644 --- a/gamutrf/specgram.py +++ b/gamutrf/specgram.py @@ -13,9 +13,8 @@ from gamutrf.utils import get_nondot_files from gamutrf.utils import is_fft -from gamutrf.utils import parse_filename from gamutrf.utils import replace_ext -from gamutrf.sample_reader import read_recording +from gamutrf.sample_reader import read_recording, parse_filename def stride_windows(x, n, noverlap=0, axis=0): @@ -273,22 +272,16 @@ def plot_spectrogram( def process_recording(args, recording): - ( - _epoch_time, - freq_center, - sample_rate, - sample_dtype, - sample_len, - _sample_type, - _sample_bits, - ) = parse_filename(recording) + meta = parse_filename(recording) + center_frequency = meta["center_frequency"] + sample_rate = int(meta["sample_rate"]) + sample_len = int(meta["sample_len"]) + sample_dtype = meta["sample_dtype"] spectrogram_filename = replace_ext(recording, args.iext, all_ext=True) if args.skip_exist and os.path.exists(spectrogram_filename): print(f"skipping {recording}") return if is_fft(recording): - # always in complex float format. - sample_dtype = np.dtype([("i", "float32"), ("q", "float32")]) if not args.skip_fft: print(f"skipping precomputed FFT {recording}") return @@ -310,7 +303,7 @@ def process_recording(args, recording): spectrogram_filename, args.nfft, sample_rate, - freq_center, + center_frequency, args.cmap, args.ytics, args.bare, diff --git a/gamutrf/utils.py b/gamutrf/utils.py index cd483193..d80d3449 100644 --- a/gamutrf/utils.py +++ b/gamutrf/utils.py @@ -20,7 +20,7 @@ # UHD_IMAGES_DIR=/usr/share/uhd/images ./examples/rx_samples_to_file --args num_recv_frames=1000,recv_frame_size=16360 --file test.gz --nsamps 200000000 --rate 20000000 --freq 101e6 --spb 20000000 ETTUS_ARGS = "num_recv_frames=1000,recv_frame_size=16360,type=b200" ETTUS_ANT = "TX/RX" -SAMPLE_FILENAME_RE = re.compile(r"^.+\D(\d+)_(\d+)Hz_(\d+)sps\.c*([fisu]\d+|raw).*$") +SAMPLE_FILENAME_RE = re.compile(r"^.+\D(\d+)_(\d+)Hz.*\D(\d+)sps\.c*([fisu]\d+|raw).*$") SAMPLE_DTYPES = { "i8": ("= min_freq) - and ((freq_center + (sample_rate / 2)) <= max_freq) + ((center_frequency + (sample_rate / 2)) >= min_freq) + and ((center_frequency + (sample_rate / 2)) <= max_freq) ) or ( - ((freq_center - (sample_rate / 2)) >= min_freq) - and ((freq_center - (sample_rate / 2)) <= max_freq) + ((center_frequency - (sample_rate / 2)) >= min_freq) + and ((center_frequency - (sample_rate / 2)) <= max_freq) ) ) and f not in processed_files: if ( not processing_batch - or processing_batch[-1]["freq_center"] < freq_center + or processing_batch[-1]["center_frequency"] < center_frequency ): processing_batch.append(file_info) else: @@ -166,12 +166,12 @@ def main(): filename = file_info["filename"] sample_dtype = file_info["sample_dtype"] sample_bytes = file_info["sample_len"] - freq_center = file_info["freq_center"] + center_frequency = file_info["center_frequency"] sample_rate = file_info["sample_rate"] sample_type = file_info["sample_type"] sample_bits = file_info["sample_bits"] timestamp = file_info["timestamp"] - # filename, freq_center, sample_rate, sample_dtype, sample_bytes, sample_type, sample_bits, _, timestamp = parse_filename(f) + # filename, center_frequency, sample_rate, sample_dtype, sample_bytes, sample_type, sample_bits, _, timestamp = parse_filename(f) samples = read_samples( filename, sample_dtype, sample_bytes, seek_bytes=0, nfft=nfft, n=n @@ -188,17 +188,17 @@ def main(): return_onesided=False, ) freq_bins = np.fft.fftshift(freq_bins) - # print(f"{freq_bins.shape=}{freq_center + freq_bins=}") + # print(f"{freq_bins.shape=}{center_frequency + freq_bins=}") idx = np.array( [ round((item - min_freq) / freq_resolution) - for item in freq_bins + freq_center + for item in freq_bins + center_frequency ] ).astype(int) # print(f"{idx=}") # print(f"{np.where(idx>=0)}") - # print(f"{(freq_bins+freq_center)[np.where(idx>=0)]=}") + # print(f"{(freq_bins+center_frequency)[np.where(idx>=0)]=}") # print(f"{freq_bins.shape=}, {S.shape=}") S = np.fft.fftshift(S, axes=0) diff --git a/tests/test_birdseye_rssi.py b/tests/test_birdseye_rssi.py index 8680cb40..5a47aa73 100644 --- a/tests/test_birdseye_rssi.py +++ b/tests/test_birdseye_rssi.py @@ -26,8 +26,10 @@ def __init__(self, filename): class BirdseyeRSSITestCase(unittest.TestCase): def test_birdseye_smoke(self): with tempfile.TemporaryDirectory() as tmpdir: - filename = os.path.join(tmpdir, "gamutrf_recording1_1000Hz_1000sps.raw") - subprocess.check_call(["dd", "if=/dev/zero", "of=" + filename, "bs=1M", "count=1"]) + filename = os.path.join(tmpdir, "gamutrf_recording1_1000Hz_1000sps.raw") + subprocess.check_call( + ["dd", "if=/dev/zero", "of=" + filename, "bs=1M", "count=1"] + ) tb = BirdsEyeRSSI(FakeArgs(filename), 1e3, 1e3) tb.start() time.sleep(10) diff --git a/tests/test_sample_reader.py b/tests/test_sample_reader.py index 0d9a37a7..6657c487 100644 --- a/tests/test_sample_reader.py +++ b/tests/test_sample_reader.py @@ -4,8 +4,7 @@ import unittest import numpy as np -from gamutrf.sample_reader import read_recording -from gamutrf.utils import parse_filename +from gamutrf.sample_reader import read_recording, parse_filename class ReadRecordingTestCase(unittest.TestCase): @@ -14,28 +13,26 @@ def test_read_recording(self): recording = os.path.join( str(tempdir), "testrecording_123_100Hz_1000sps.ci16" ) - # parse filename determines sample dtype, et al from filename - ( - _epoch_time, - freq_center, - sample_rate, - sample_dtype, - sample_len, - sample_type, - sample_bits, - ) = parse_filename(recording) - samples = chr(0) * int(sample_len * sample_rate) + meta = parse_filename(recording) + samples = chr(0) * int(meta["sample_len"] * meta["sample_rate"]) with open(recording, "wb") as f: f.write(samples.encode("utf8")) - self.assertEqual(100, freq_center) - self.assertEqual(1000, sample_rate) - self.assertEqual("signed-integer", sample_type) - self.assertEqual(16, sample_bits) - self.assertEqual(np.dtype([("i", "