Merge branch 'IQTLabs:main' into main

.github/workflows/docker-test.yml

@@ -29,9 +29,10 @@ jobs:
         docker build -f Dockerfile.base . -t iqtlabs/gamutrf-base:latest
         cd ..
         docker build -f Dockerfile . -t iqtlabs/gamutrf:latest
+        docker build -f docker/Dockerfile.torchsig . -t iqtlabs/gamutrf-torchsig:latest
         docker run -t iqtlabs/gamutrf:latest gamutrf-scan --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-sigfinder --help
-        docker run -t iqtlabs/gamutrf:latest gamutrf-api --help
+        docker run -t iqtlabs/gamutrf:latest gamutrf-worker --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-samples2raw --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-freqxlator --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-waterfall --help

.github/workflows/docker.yml

@@ -84,3 +84,12 @@ jobs:
         push: true
         tags: iqtlabs/gamutrf:${{ steps.change_version.outputs.VERSION }}
       if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
+    - name: Build and push gamutrf-torchsig
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: docker/Dockerfile.torchsig
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: iqtlabs/gamutrf-torchsig:${{ steps.change_version.outputs.VERSION }}
+      if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'

augment/augment.py

@@ -0,0 +1,142 @@
+#!/usr/bin/python3
+
+from argparse import ArgumentParser
+import os
+import sigmf
+from sigmf import SigMFFile
+from sigmf.utils import get_data_type_str
+import numpy as np
+import torchsig.transforms.transforms as ST
+from torchsig.transforms.functional import (
+    to_distribution,
+    uniform_continuous_distribution,
+    uniform_discrete_distribution,
+)
+from torchsig.utils.types import SignalData, SignalDescription
+from gamutrf.sample_reader import read_recording
+from gamutrf.waterfall_samples import parse_filename
+
+
+def make_signal(samples, sample_rate, center_frequency):
+    num_iq_samples = samples.shape[0]
+    desc = SignalDescription(
+        sample_rate=sample_rate,
+        num_iq_samples=num_iq_samples,
+        center_frequency=center_frequency,
+    )
+    # TODO: subclass SignalData with alternate constructor that can take just numpy array
+    signal = SignalData(samples.tobytes(), np.float32, np.complex128, desc)
+    return signal
+
+
+def get_nosigmf_file(filename):
+    meta = parse_filename(filename)
+    sample_rate = meta["sample_rate"]
+    sample_dtype = meta["sample_dtype"]
+    sample_len = meta["sample_len"]
+    center_frequency = meta["freq_center"]
+    samples = None
+    for samples_buffer in read_recording(
+        filename, sample_rate, sample_dtype, sample_len, max_sample_secs=None
+    ):
+        if samples is None:
+            samples = samples_buffer
+        else:
+            samples = np.concatenate([samples, samples_buffer])
+    signal = make_signal(samples, sample_rate, center_frequency)
+    return filename, signal
+
+
+def get_signal(filename):
+    if not os.path.exists(filename):
+        raise FileNotFoundError(filename)
+    meta_ext = filename.find(".sigmf-meta")
+    if meta_ext == -1:
+        return get_nosigmf_file(filename)
+
+    meta = sigmf.sigmffile.fromfile(filename)
+    data_filename = filename[:meta_ext]
+    meta.set_data_file(data_filename)
+    # read_samples() always converts to host cf32.
+    samples = meta.read_samples()
+    global_meta = meta.get_global_info()
+    sample_rate = global_meta["core:sample_rate"]
+    sample_type = global_meta["core:datatype"]
+    captures_meta = meta.get_captures()
+    center_frequency = None
+    if captures_meta:
+        center_frequency = captures_meta[0].get("core:frequency", None)
+    signal = make_signal(samples, sample_rate, center_frequency)
+    return data_filename, signal
+
+
+def write_signal(filename, signal, transforms_text):
+    first_desc = signal.signal_description[0]
+    signal.iq_data = signal.iq_data.astype(np.complex64)
+    signal.iq_data.tofile(filename)
+
+    new_meta = SigMFFile(
+        data_file=filename,
+        global_info={
+            SigMFFile.DATATYPE_KEY: get_data_type_str(signal.iq_data),
+            SigMFFile.SAMPLE_RATE_KEY: first_desc.sample_rate,
+            SigMFFile.VERSION_KEY: sigmf.__version__,
+            SigMFFile.DESCRIPTION_KEY: transforms_text,
+        },
+    )
+    new_meta.add_capture(
+        0,
+        metadata={
+            SigMFFile.FREQUENCY_KEY: first_desc.center_frequency,
+        },
+    )
+    new_meta.tofile(".".join([filename, "sigmf-meta"]))
+
+
+def augment(signal, filename, output_dir, n, transforms_text):
+    # TODO: sadly, due to Torchsig complexity, literal_eval can't be used.
+    transforms = eval(transforms_text)  # nosec
+    i = 0
+    base_augment_name = os.path.basename(filename)
+    dot = base_augment_name.find(".")
+    if dot != -1:
+        base_augment_name = base_augment_name[:dot]
+    for _ in range(n):
+        while True:
+            augment_name = os.path.join(
+                output_dir, f"augmented-{i}-{base_augment_name}"
+            )
+            if not os.path.exists(augment_name):
+                break
+            i += 1
+        new_signal = transforms(signal)
+        write_signal(augment_name, new_signal, transforms_text)
+
+
+def argument_parser():
+    parser = ArgumentParser(
+        description="Run transforms on a recording from https://github.com/TorchDSP/torchsig/blob/main/torchsig/transforms/transforms.py"
+    )
+    parser.add_argument(
+        "filename",
+        type=str,
+        help="sigMF file or gamutRF zst recording",
+    )
+    parser.add_argument("outdir", type=str, help="output directory")
+    parser.add_argument("n", type=int, help="number of augmentation passes")
+    parser.add_argument(
+        "transforms",
+        type=str,
+        help="transforms to eval, e.g. ST.Compose([ST.AddNoise((-40, -20)),ST.RandomPhaseShift(uniform_continuous_distribution(-1, 1))]) (use quotes)",
+    )
+    return parser
+
+
+def main():
+    options = argument_parser().parse_args()
+    data_filename, signal = get_signal(options.filename)
+    augment(signal, data_filename, options.outdir, options.n, options.transforms)
+
+
+if __name__ == "__main__":
+    main()

bin/gamutrf

@@ -9,7 +9,7 @@ Options:
     -h,  help          print this help
     -i,  install       install GamutRF repo, optionally supply a version, tag, branch, or tarball
     -l,  logs          tail GamutRF logs
-    -r,  run           specify GamutRF tool to run (and any additional args for the tools), options include: 'api', 'freqxlator', 'samples2raw', 'scan', 'sigfinder', 'specgram'
+    -r,  run           specify GamutRF tool to run (and any additional args for the tools), options include: 'worker', 'freqxlator', 'samples2raw', 'scan', 'sigfinder', 'specgram'
     -R,  restart       specify 'orchestrator' or 'worker' to restart
     -s,  start         specify 'orchestrator' or 'worker' to start
     -S,  stop          specify 'orchestrator' or 'worker' to stop
@@ -136,7 +136,7 @@ function check_args()
                 ;;
             -r|run)
                 if [ -z "$2" ]; then
-                    echo "Specify 'api', 'freqxlator', 'samples2raw', 'scan', 'sigfinder', or 'specgram' to run"
+                    echo "Specify 'worker', 'freqxlator', 'samples2raw', 'scan', 'sigfinder', or 'specgram' to run"
                     exit
                 fi
                 docker run -it -v "$VOLUME_DIR":/data iqtlabs/gamutrf:"$VERS" gamutrf-"$2" "${@:3}"

docker/Dockerfile.torchsig

@@ -0,0 +1,23 @@
+FROM ubuntu:22.04 as torchsig-builder
+WORKDIR /root
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential ca-certificates cmake git python3-pip python3-dev && pip install -U pip
+# Cause torch CPU only to be installed, no cuda dependencies
+RUN pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
+RUN git clone https://github.com/TorchDSP/torchsig -b v0.4.1
+WORKDIR /root/torchsig
+RUN sed -i -E "s/torch==[0-9\.]+/torch/g" pyproject.toml
+RUN pip install .
+
+FROM iqtlabs/gamutrf:latest
+WORKDIR /root
+ENV DEBIAN_FRONTEND noninteractive
+# TODO: find a better way cherrypick just Torchsig itself, without Torch et al. Torchsig transforms have dependencies on
+# Torch, even though we don't need Torch for the standalone transforms we want.
+COPY --from=torchsig-builder /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages
+RUN python3 -c "from torchsig.transforms import transforms"
+RUN python3 -c "from gamutrf import grscan"
+COPY augment/augment.py /root/augment.py
+RUN dd if=/dev/zero of=/tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000sps.raw bs=8 count=1000 && /root/augment.py /tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000sps.raw /tmp 1 "ST.Compose([ST.Identity()])" && diff -b /tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000sps.raw /tmp/augmented-0-gamutrf_recording_ettus__gain40_1_1Hz_1000sps && rm -f /tmp/gamutrf*
+
+ENTRYPOINT ["/root/augment.py"]

docs/BUILD.md

@@ -295,7 +295,7 @@ If run in `recorder` mode (the default) no changes on the worker are needed, but
 
 If run in `RSSI` mode the `worker.yml` file under the gamutrf directory needs to be updated to include the following options:
 ```
-  gamutrf-api:
+  gamutrf-worker:
     restart: always
     image: iqtlabs/gamutrf:latest
     networks:
@@ -319,7 +319,7 @@ If run in `RSSI` mode the `worker.yml` file under the gamutrf directory needs to
       - nice
       - '-n'
       - '-19'
-      - gamutrf-api
+      - gamutrf-worker
       - --no-agc
       - --rxb=62914560
       - '--gain=${GAIN}'

gamutrf/__main__.py

@@ -1,5 +1,5 @@
 """Main entrypoint for GamutRF"""
-from gamutrf.api import main as api_main
+from gamutrf.worker import main as worker_main
 from gamutrf.compress_dirs import main as compress_dirs_main
 from gamutrf.freqxlator import main as freqxlator_main
 from gamutrf.samples2raw import main as samples2raw_main
@@ -9,9 +9,9 @@
 from gamutrf.waterfall import main as waterfall_main
 
 
-def api():
-    """Entrypoint for API"""
-    api_main()
+def worker():
+    """Entrypoint for worker"""
+    worker_main()
 
 
 def compress_dirs():

pyproject.toml

@@ -16,7 +16,7 @@ falcon = "3.1.1"
 falcon-cors = "1.1.7"
 findpeaks = "2.5.5"
 gpsd-py3 = "0.3.0"
-httpx = "0.25.0"
+httpx = "0.25.1"
 Jinja2 = "3.1.2"
 matplotlib = "3.8.1"
 numpy = "1.26.1"
@@ -39,7 +39,7 @@ pyzmq = "^25.1.0"
 [tool.poetry.dev-dependencies]
 attr = "0.3.2"
 attrs = "23.1.0"
-black = "23.10.1"
+black = "23.11.0"
 docker = "6.1.3"
 pylint = "3.0.2"
 pytest = "7.4.3"
@@ -48,7 +48,7 @@ pytype = "2023.10.31"
 pdbpp = "^0.10.3"
 
 [tool.poetry.scripts]
-gamutrf-api = 'gamutrf.__main__:api'
+gamutrf-worker = 'gamutrf.__main__:worker'
 gamutrf-freqxlator = 'gamutrf.__main__:freqxlator'
 gamutrf-samples2raw = 'gamutrf.__main__:samples2raw'
 gamutrf-scan = 'gamutrf.__main__:scan'

tests/test_api.py

@@ -2,7 +2,7 @@
 import pytest
 from falcon import testing
 
-from gamutrf import api
+from gamutrf import worker
 
 
 class FakeArgs:
@@ -32,7 +32,7 @@ def __init__(self):
 
 @pytest.fixture(scope="module")
 def client():
-    app = api.API(FakeArgs())
+    app = worker.API(FakeArgs())
     return testing.TestClient(app.app)
 
 
@@ -46,21 +46,21 @@ def test_routes(client):
 
 
 def test_report_rssi():
-    app = api.API(FakeArgs())
+    app = worker.API(FakeArgs())
     app.report_rssi({"center_freq": 1e6}, -35, time.time())
 
 
 def test_serve_recording():
-    app = api.API(FakeArgs())
+    app = worker.API(FakeArgs())
     app.q.put({"center_freq": 1e6, "sample_count": 1e6})
     app.serve_recording(app.record)
 
 
 def test_serve_rssi():
-    app = api.API(FakeArgs())
+    app = worker.API(FakeArgs())
     app.q.put({"center_freq": 1e6, "sample_count": 1e6, "sample_rate": 1e6})
     app.serve_rssi()
 
 
 def test_argument_parse():
-    api.argument_parser()
+    worker.argument_parser()