From 2a7abd02a6696b092263af022664e4ba25b795d7 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Thu, 14 Dec 2023 20:22:31 +0000 Subject: [PATCH] Remove samples2raw (not as useful now that we have SigMF and augment), and freqxlator (also can be done with augment). --- .github/workflows/ci-test.yml | 9 --- .github/workflows/docker-test.yml | 2 - bin/gamutrf | 4 +- docs/2-SYSTEM_OVERVIEW.md | 4 +- docs/4-OPERATION.md | 20 ----- gamutrf/__main__.py | 12 --- gamutrf/freqxlator.py | 124 ------------------------------ gamutrf/samples2raw.py | 77 ------------------- pyproject.toml | 2 - tests/test_freqxlator.py | 17 ---- tests/test_main.py | 16 ---- tests/test_samples2raw.py | 32 -------- 12 files changed, 4 insertions(+), 315 deletions(-) delete mode 100644 gamutrf/freqxlator.py delete mode 100644 gamutrf/samples2raw.py delete mode 100644 tests/test_freqxlator.py delete mode 100644 tests/test_samples2raw.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 88410e8c..e255121e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -52,15 +52,6 @@ jobs: poetry install --no-interaction && \ poetry install --no-interaction -C utils/mavlink-api && \ sudo dpkg -r --force-depends python3-numpy - - name: test gamutrf-samples2raw - env: - PYTHONPATH: /usr/local/lib/python3.10/dist-packages:/usr/lib/python3/dist-packages - run: | - gamutrf-samples2raw --help && \ - dd if=/dev/zero of=/tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000000sps.s16 bs=4 count=1000000 && \ - gamutrf-samples2raw /tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000000sps.s16 --outfmt=float && \ - dd if=/dev/zero of=/tmp/floats bs=8 count=1000000 && \ - diff /tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000000sps.raw /tmp/floats - name: Code Quality - Black run: | poetry run black gamutrf --check diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 52e3aad3..d7b18c01 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -36,9 +36,7 @@ jobs: docker build -f Dockerfile . -t iqtlabs/gamutrf:latest docker build -f docker/Dockerfile.torchsig . -t iqtlabs/gamutrf-torchsig:latest docker run -t iqtlabs/gamutrf:latest gamutrf-compress_dirs --help - docker run -t iqtlabs/gamutrf:latest gamutrf-freqxlator --help docker run -t iqtlabs/gamutrf:latest gamutrf-offline --help - docker run -t iqtlabs/gamutrf:latest gamutrf-samples2raw --help docker run -t iqtlabs/gamutrf:latest gamutrf-scan --help docker run -t iqtlabs/gamutrf:latest gamutrf-sigfinder --help docker run -t iqtlabs/gamutrf:latest gamutrf-waterfall --help diff --git a/bin/gamutrf b/bin/gamutrf index 77211bea..513bfdfd 100644 --- a/bin/gamutrf +++ b/bin/gamutrf @@ -9,7 +9,7 @@ Options: -h, help print this help -i, install install GamutRF repo, optionally supply a version, tag, branch, or tarball -l, logs tail GamutRF logs - -r, run specify GamutRF tool to run (and any additional args for the tools), options include: 'worker', 'freqxlator', 'samples2raw', 'scan', 'sigfinder', 'specgram' + -r, run specify GamutRF tool to run (and any additional args for the tools), options include: 'worker', 'scan', 'sigfinder', 'specgram' -R, restart specify 'orchestrator' or 'worker' to restart -s, start specify 'orchestrator' or 'worker' to start -S, stop specify 'orchestrator' or 'worker' to stop @@ -136,7 +136,7 @@ function check_args() ;; -r|run) if [ -z "$2" ]; then - echo "Specify 'worker', 'freqxlator', 'samples2raw', 'scan', 'sigfinder', or 'specgram' to run" + echo "Specify 'worker', 'scan', 'sigfinder', or 'specgram' to run" exit fi docker run -it -v "$VOLUME_DIR":/data iqtlabs/gamutrf:"$VERS" gamutrf-"$2" "${@:3}" diff --git a/docs/2-SYSTEM_OVERVIEW.md b/docs/2-SYSTEM_OVERVIEW.md index fd4ed294..161de777 100644 --- a/docs/2-SYSTEM_OVERVIEW.md +++ b/docs/2-SYSTEM_OVERVIEW.md @@ -81,7 +81,7 @@ The generic route for data during operation starts with the gamutrf-scan contain Sigfinder can task the api container which is running on a worker node to investigate a signal. The worker tunes to listen to the tasked frequency and either calculates the associated RSSI or makes a recording of the signal. In RSSI mode, the worker interrogates the gpsd service and compass container running on the orchestrator to get position and heading, then publishes location, heading, and RSSI to the MQTT broker to be consumed by Birdseye. Birdseye then uses this information to geolocate the target signal. -Orchestrator.yml and worker.yml are the main control points for docker-compose to start the GamutRF system. These scripts spin up the iqtlabs/gamutrf Docker container and run the appropriate gamutrf- python command. These files should be modified for the specific orchestrator and worker combination. Some common parameters for the gamut-scan and gamutrf-sigfinder are in the GamutRF [operation guide](./4-OPERATION.md).md. The repository also provides a few tools such as samples2raw, freqxlator, freespacer, compress_dirs, etc that can be used for post processing of data. These tools are included in the GamutRF python module so it is suggested to use them via iqtlabs/gamutrf docker container. +Orchestrator.yml and worker.yml are the main control points for docker-compose to start the GamutRF system. These scripts spin up the iqtlabs/gamutrf Docker container and run the appropriate gamutrf- python command. These files should be modified for the specific orchestrator and worker combination. Some common parameters for the gamut-scan and gamutrf-sigfinder are in the GamutRF [operation guide](./4-OPERATION.md).md. The repository also provides a few tools such as freespacer, compress_dirs, etc that can be used for post processing of data. These tools are included in the GamutRF python module so it is suggested to use them via iqtlabs/gamutrf docker container. ## Graphical User Interface @@ -122,4 +122,4 @@ Initially we used the received signal strength along with simple propagation mod We take a moving average of this value to smooth it out as it can jump around quite a bit and we are most interested in the slower moving trend of the value. This helps to estimate distance from the target. Figure 7 below shows the portion of the GNU Radio flow graph that estimates the RSSI value. -Birdseye (RF geolocalization tool) is also located on the IQT Labs Github at https://github.com/iqtlabs/birdseye. The repository contains instructions for using the Birdseye tool as well as different methodologies for ML-enabled localization. Birdseye can be used in conjunction with GamutRF or as a standalone tool. When using with the GamutRF system, the instructions for integration are in the Build.md file which detail how to run Birdseye as a systemd service. \ No newline at end of file +Birdseye (RF geolocalization tool) is also located on the IQT Labs Github at https://github.com/iqtlabs/birdseye. The repository contains instructions for using the Birdseye tool as well as different methodologies for ML-enabled localization. Birdseye can be used in conjunction with GamutRF or as a standalone tool. When using with the GamutRF system, the instructions for integration are in the Build.md file which detail how to run Birdseye as a systemd service. diff --git a/docs/4-OPERATION.md b/docs/4-OPERATION.md index 1d39f55d..58a245bd 100644 --- a/docs/4-OPERATION.md +++ b/docs/4-OPERATION.md @@ -108,14 +108,6 @@ gamutRF provides a tool to convert a recording or directory of recordings into a Use the ```--help``` option to change how the spectogram is generated (for example, to change the sample rate). -### Translating recordings to "gnuradio" format - -Most SDR tools by convention take an uncompressed raw binary file as input, of [gnuradio type complex](https://blog.sdr.hu/grblocks/types.html). The user must explicitly specify to most SDR tools what sample rate the file was made at to correctly process it. gamutRF provides a tool that converts a gamutRF I/Q recording (which may be compressed) to an uncompressed binary file. For example: - -``` -docker run -v /tmp:/tmp -ti iqtlabs/gamutrf gamutrf-samples2raw /tmp/gamutrf_recording_ettus_directional_gain70_1234_100000000Hz_20971520sps.s16.zst -``` - ### Reviewing a recording interactively in gqrx [gqrx](https://gqrx.dk/) is a multiplatform open source tool that allows some basic SDR operations like visualizing or audio demodulating an I/Q sample recording (see the [github releases page](https://github.com/gqrx-sdr/gqrx/releases), for a MacOS .dmg file). To use gqrx with a gamutRF recording, first translate the recording to gnuradio format (see above). Then open gqrx. @@ -128,22 +120,10 @@ docker run -v /tmp:/tmp -ti iqtlabs/gamutrf gamutrf-samples2raw /tmp/gamutrf_rec * Set ```Decimation``` to None. * Finally select ```OK``` and then ```play``` from the gqrx interface to watch the recording play. -### Reducing recording sample rate - -You may want to reduce the sample rate of a recording or re-center it with respect to frequency (e.g. to use another demodulator tool that doesn't support a high sample rate). gamutRF provides the ```freqxlator``` tool to do this. - -* Translate your gamutRF recording to gnuradio format (see above). -* Use ```freqxlator``` to create a new recording at a lower sample rate, potentially with a different center frequency. - -For example, to reduce a recording made with gamutRF's default sample rate to 1/10th the rate while adjusting the center frequency down by 1MHz, use: - -```docker run -ti iqtlabs/gamutrf gamutrf-freqxlator --samp-rate 20971520 --center -1e6 --dec 10 gamutrf_recording_gain70_1234_100000000Hz_20971520sps.raw gamutrf_recording_gain70_1234_100000000Hz_2097152sps.raw``` - ### Demodulating AM/FM audio from a recording gamutRF provides a tool to demodulate AM/FM audio from a recording as an example use case. -* Use the ```freqxlator``` tool to make a new recording at no more than 1Msps and has the frequency to be demodulated centered. * Use the ```airspyfm``` tool to demodulate audio to a WAV file. For example, to decode an FM recording which must be at the center frequency of a recording: diff --git a/gamutrf/__main__.py b/gamutrf/__main__.py index ad144461..af3866fe 100644 --- a/gamutrf/__main__.py +++ b/gamutrf/__main__.py @@ -1,8 +1,6 @@ """Main entrypoint for GamutRF""" from gamutrf.compress_dirs import main as compress_dirs_main -from gamutrf.freqxlator import main as freqxlator_main from gamutrf.offline import main as offline_main -from gamutrf.samples2raw import main as samples2raw_main from gamutrf.scan import main as scan_main from gamutrf.sigfinder import main as sigfinder_main from gamutrf.specgram import main as specgram_main @@ -15,21 +13,11 @@ def compress_dirs(): compress_dirs_main() -def freqxlator(): - """Entrypoint for freqxlator""" - freqxlator_main() - - def offline(): """Entrypoint for offline""" offline_main() -def samples2raw(): - """Entrypoint for samples2raw""" - samples2raw_main() - - def scan(): """Entrypoint for scan""" scan_main() diff --git a/gamutrf/freqxlator.py b/gamutrf/freqxlator.py deleted file mode 100644 index 82952d43..00000000 --- a/gamutrf/freqxlator.py +++ /dev/null @@ -1,124 +0,0 @@ -# Extract and decimate narrowband signal from wideband signal -# -# Example of decoding FM signal centered at 98.1MHz, from 20MHz wideband recording centered at 100MHz recorded at 20Msps. -# -# Convert int16 recording to complex float: -# -# $ samples2raw.py gamutrf_recording1643863029_100000000Hz_20000000sps.s16.gz -# -# Select new center -1.9MHz, from original center and downsample to 1Msps -# -# $ freqxlator.py --samp-rate=20e6 --center=-1.9e6 --dec=20 --infile gamutrf_recording1643863029_100000000Hz_20000000sps.raw --outfile fm.raw -# -# Decode 1Msps recording as FM. -# -# $ airspy-fmradion -t filesource -c srate=1000000,raw,format=FLOAT,filename=fm.raw -W fm.wav -import argparse -import sys - -try: - from gnuradio import blocks # pytype: disable=import-error - from gnuradio import eng_notation # pytype: disable=import-error - from gnuradio import gr # pytype: disable=import-error - from gnuradio.eng_arg import eng_float # pytype: disable=import-error - from gnuradio.filter import firdes - from gnuradio.filter import freq_xlating_fir_filter_ccc - from pmt import PMT_NIL # pytype: disable=import-error -except ModuleNotFoundError as err: - print( - "Run from outside a supported environment, please run via Docker (https://github.com/IQTLabs/gamutRF#readme): %s" - % err - ) - sys.exit(1) - - -class FreqXLator(gr.top_block): - def __init__(self, samp_rate, center, transitionbw, dec, infile, outfile): - gr.top_block.__init__(self, "freqxlator", catch_exceptions=True) - - self.samp_rate = samp_rate - self.center = center - self.transitionbw = transitionbw - self.dec = dec - self.infile = infile - self.outfile = outfile - - self.freq_xlating_fir_filter_xxx_0 = freq_xlating_fir_filter_ccc( - self.dec, self._get_taps(), self.center, self.samp_rate - ) - self.blocks_file_source_0 = blocks.file_source( - gr.sizeof_gr_complex, self.infile, False, 0, 0 - ) # pylint: disable=no-member - self.blocks_file_source_0.set_begin_tag(PMT_NIL) - self.blocks_file_sink_0 = blocks.file_sink( - gr.sizeof_gr_complex, self.outfile, False - ) # pylint: disable=no-member - self.blocks_file_sink_0.set_unbuffered(False) - - self.connect( - (self.blocks_file_source_0, 0), (self.freq_xlating_fir_filter_xxx_0, 0) - ) - self.connect( - (self.freq_xlating_fir_filter_xxx_0, 0), (self.blocks_file_sink_0, 0) - ) - - def _get_taps(self): - return firdes.complex_band_pass( - 1, - self.samp_rate, - -self.samp_rate / (2 * self.dec), - self.samp_rate / (2 * self.dec), - self.transitionbw, - ) - - -def argument_parser(): - parser = argparse.ArgumentParser( - "Extract and decimate narrowband signal from wideband signal" - ) - parser.add_argument("infile", type=str, help="Input file (complex I/Q format)") - parser.add_argument("outfile", type=str, help="Output file (complex I/Q format)") - parser.add_argument( - "--samp-rate", - dest="samp_rate", - type=eng_float, - default=eng_notation.num_to_str(float(20e6)), - help="Set samp_rate [default=%(default)r]", - ) - parser.add_argument( - "--center", - dest="center", - type=eng_float, - default=eng_notation.num_to_str(float(-1e6)), - help="Offset to new center frequency [default=%(default)r]", - ) - parser.add_argument( - "--transitionbw", - dest="transitionbw", - type=eng_float, - default=eng_notation.num_to_str(float(10e3)), - help="Filter transmission bandwidth [default=%(default)r]", - ) - parser.add_argument( - "--dec", - dest="dec", - type=int, - default=20, - help="Decimation [default=%(default)r]", - ) - return parser - - -def main(): - parser = argument_parser() - options = parser.parse_args() - block = FreqXLator( - options.samp_rate, - options.center, - options.transitionbw, - options.dec, - options.infile, - options.outfile, - ) - block.start() - block.wait() diff --git a/gamutrf/samples2raw.py b/gamutrf/samples2raw.py deleted file mode 100644 index ea9f0088..00000000 --- a/gamutrf/samples2raw.py +++ /dev/null @@ -1,77 +0,0 @@ -import argparse -import subprocess - -from gamutrf.sample_reader import parse_filename -from gamutrf.utils import replace_ext - - -def make_procs_args(sample_filename, outfmt): - procs_args = [] - out_filename = sample_filename - - if sample_filename.endswith(".gz"): - procs_args.append(["gunzip", "-c", sample_filename]) - out_filename = replace_ext(out_filename, "") - elif sample_filename.endswith(".zst"): - procs_args.append(["zstdcat", sample_filename]) - out_filename = replace_ext(out_filename, "") - - meta = parse_filename(out_filename) - sample_rate = meta["sample_rate"] - in_format = meta["sample_type"] - in_bits = meta["sample_bits"] - print(meta) - out_filename = replace_ext(out_filename, "raw", all_ext=True) - sox_in = sample_filename - if procs_args: - sox_in = "-" - procs_args.append( - [ - "sox", - "-D", # disable dithering. - "-t", - "raw", - "-r", - str(sample_rate), - "-c", - "1", - "-b", - str(in_bits), - "-e", - in_format, - sox_in, - "-e", - outfmt, - out_filename, - ] - ) - return procs_args - - -def run_procs(procs_args): - procs = [] - for proc_args in procs_args: - stdin = None - if procs: - stdin = procs[-1].stdout - procs.append(subprocess.Popen(proc_args, stdout=subprocess.PIPE, stdin=stdin)) - for proc in procs[:-1]: - if proc.stdout is not None: - proc.stdout.close() - procs[-1].communicate() - - -def argument_parser(): - parser = argparse.ArgumentParser( - description="Convert (possibly compressed) sample recording to a float raw file (gnuradio style)" - ) - parser.add_argument("samplefile", default="", help="sample file to read") - parser.add_argument("--outfmt", default="float", help="output format") - return parser - - -def main(): - parser = argument_parser() - args = parser.parse_args() - procs_args = make_procs_args(args.samplefile, args.outfmt) - run_procs(procs_args) diff --git a/pyproject.toml b/pyproject.toml index 0c4a0cc0..d4ca438e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,7 @@ pytype = "2023.12.8" [tool.poetry.scripts] gamutrf-compress_dirs = 'gamutrf.__main__:compress_dirs' -gamutrf-freqxlator = 'gamutrf.__main__:freqxlator' gamutrf-offline= 'gamutrf.__main__:offline' -gamutrf-samples2raw = 'gamutrf.__main__:samples2raw' gamutrf-scan = 'gamutrf.__main__:scan' gamutrf-sigfinder = 'gamutrf.__main__:sigfinder' gamutrf-specgram = 'gamutrf.__main__:specgram' diff --git a/tests/test_freqxlator.py b/tests/test_freqxlator.py deleted file mode 100644 index f3778e55..00000000 --- a/tests/test_freqxlator.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python3 -import unittest - -from gamutrf.freqxlator import FreqXLator -from gamutrf.freqxlator import argument_parser - - -class FreqXlatorTestCase(unittest.TestCase): - def test_freqxlator_smoke(self): - FreqXLator(1e3, 100e3, 10e3, 10, "/dev/null", "/dev/null") - - def test_argument_parser(self): - argument_parser() - - -if __name__ == "__main__": # pragma: no cover - unittest.main() diff --git a/tests/test_main.py b/tests/test_main.py index 06d38c30..584db1a9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,8 +2,6 @@ import sys from gamutrf.__main__ import worker -from gamutrf.__main__ import freqxlator -from gamutrf.__main__ import samples2raw from gamutrf.__main__ import scan from gamutrf.__main__ import sigfinder from gamutrf.__main__ import specgram @@ -19,20 +17,6 @@ def test_main_worker(): assert pytest_wrapped_e.value.code == 0 -def test_main_freqxlator(): - with pytest.raises(SystemExit) as pytest_wrapped_e: - freqxlator() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - - -def test_main_samples2raw(): - with pytest.raises(SystemExit) as pytest_wrapped_e: - samples2raw() - assert pytest_wrapped_e.type == SystemExit - assert pytest_wrapped_e.value.code == 0 - - def test_main_scan(): with pytest.raises(SystemExit) as pytest_wrapped_e: scan() diff --git a/tests/test_samples2raw.py b/tests/test_samples2raw.py deleted file mode 100644 index aef0cb02..00000000 --- a/tests/test_samples2raw.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 - -import tempfile -import os -import unittest - -import numpy as np - -from gamutrf.samples2raw import make_procs_args, run_procs, argument_parser - - -class Samples2RawTestCase(unittest.TestCase): - def test_argument_parser(self): - argument_parser() - - def test_s2r(self): - with tempfile.TemporaryDirectory() as tempdir: - base_test_name = "gamutrf_recording1_1Hz_100sps" - test_file_name = os.path.join(tempdir, ".".join((base_test_name, "i16"))) - out_file_name = os.path.join(tempdir, ".".join((base_test_name, "raw"))) - test_data = np.int16([-(2**15)] * int(1e2 * 2)) - test_float_data = np.float32([-1] * int(1e2 * 2)) - test_data.tofile(test_file_name) - run_procs(make_procs_args(test_file_name, "float")) - converted_data = np.fromfile(out_file_name, dtype="