From ff8faf94cc7300618d20a71cdb716e7ec2d12803 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Mon, 15 Jul 2024 13:01:12 +1200 Subject: [PATCH] remove torchsig/augment. --- .github/workflows/docker-test.yml | 1 - .github/workflows/docker.yml | 9 --- augment/augment.py | 101 ------------------------------ docker/Dockerfile.torchsig | 28 --------- 4 files changed, 139 deletions(-) delete mode 100755 augment/augment.py delete mode 100644 docker/Dockerfile.torchsig diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 80d2f395..51a7c1a6 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -36,7 +36,6 @@ jobs: docker build -f docker/Dockerfile.base docker -t iqtlabs/gamutrf-base:latest docker rmi -f iqtlabs/gamutrf-vkfft:latest iqtlabs/gamutrf-driver:latest docker build -f Dockerfile . -t iqtlabs/gamutrf:latest - docker build -f docker/Dockerfile.torchsig . -t iqtlabs/gamutrf-torchsig:latest docker run -t iqtlabs/gamutrf:latest gamutrf-compress_dirs --help docker run -t iqtlabs/gamutrf:latest gamutrf-offline --help docker run -t iqtlabs/gamutrf:latest gamutrf-scan --help diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8f2b5293..7d1f7513 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -110,12 +110,3 @@ jobs: push: true tags: iqtlabs/gamutrf:${{ steps.change_version.outputs.VERSION }} if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push' - - name: Build and push gamutrf-torchsig - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.torchsig - platforms: linux/amd64,linux/arm64 - push: true - tags: iqtlabs/gamutrf-torchsig:${{ steps.change_version.outputs.VERSION }} - if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push' diff --git a/augment/augment.py b/augment/augment.py deleted file mode 100755 index cb2e7e16..00000000 --- a/augment/augment.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/python3 - -from argparse import ArgumentParser -import os -import sigmf -from sigmf import SigMFFile -from sigmf.utils import get_data_type_str -import numpy as np -import torchsig.transforms.transforms as ST -from torchsig.transforms.functional import ( - to_distribution, - uniform_continuous_distribution, - uniform_discrete_distribution, -) -from torchsig.utils.types import SignalData, SignalDescription -from gamutrf.sample_reader import read_recording, get_samples - - -def make_signal(samples, meta): - num_iq_samples = samples.shape[0] - desc = SignalDescription( - sample_rate=meta["sample_rate"], - num_iq_samples=num_iq_samples, - center_frequency=meta["center_frequency"], - ) - # TODO: subclass SignalData with alternate constructor that can take just numpy array - signal = SignalData(samples.tobytes(), np.float32, np.complex128, desc) - return signal - - -def write_signal(filename, signal, transforms_text): - first_desc = signal.signal_description[0] - signal.iq_data = signal.iq_data.astype(np.complex64) - signal.iq_data.tofile(filename) - - new_meta = SigMFFile( - data_file=filename, - global_info={ - SigMFFile.DATATYPE_KEY: get_data_type_str(signal.iq_data), - SigMFFile.SAMPLE_RATE_KEY: first_desc.sample_rate, - SigMFFile.VERSION_KEY: sigmf.__version__, - SigMFFile.DESCRIPTION_KEY: transforms_text, - }, - ) - new_meta.add_capture( - 0, - metadata={ - SigMFFile.FREQUENCY_KEY: first_desc.center_frequency, - }, - ) - new_meta.tofile(".".join([filename, "sigmf-meta"])) - - -def augment(signal, filename, output_dir, n, transforms_text): - # TODO: sadly, due to Torchsig complexity, literal_eval can't be used. - transforms = eval(transforms_text) # nosec - i = 0 - base_augment_name = os.path.basename(filename) - dot = base_augment_name.find(".") - if dot != -1: - base_augment_name = base_augment_name[:dot] - for _ in range(n): - while True: - augment_name = os.path.join( - output_dir, f"augmented-{i}-{base_augment_name}" - ) - if not os.path.exists(augment_name): - break - i += 1 - new_signal = transforms(signal) - write_signal(augment_name, new_signal, transforms_text) - - -def argument_parser(): - parser = ArgumentParser( - description="Run transforms on a recording from https://github.com/TorchDSP/torchsig/blob/main/torchsig/transforms/transforms.py" - ) - parser.add_argument( - "filename", - type=str, - help="sigMF file or gamutRF zst recording", - ) - parser.add_argument("outdir", type=str, help="output directory") - parser.add_argument("n", type=int, help="number of augmentation passes") - parser.add_argument( - "transforms", - type=str, - help="transforms to eval, e.g. ST.Compose([ST.AddNoise((-40, -20)),ST.RandomPhaseShift(uniform_continuous_distribution(-1, 1))]) (use quotes)", - ) - return parser - - -def main(): - options = argument_parser().parse_args() - data_filename, samples, meta = get_samples(options.filename) - signal = make_signal(samples, meta) - augment(signal, data_filename, options.outdir, options.n, options.transforms) - - -if __name__ == "__main__": - main() diff --git a/docker/Dockerfile.torchsig b/docker/Dockerfile.torchsig deleted file mode 100644 index 479a9502..00000000 --- a/docker/Dockerfile.torchsig +++ /dev/null @@ -1,28 +0,0 @@ -FROM ubuntu:24.04 AS torchsig-builder -ENV DEBIAN_FRONTEND=noninteractive -WORKDIR /root/.config/pip -COPY pip.conf pip.conf -WORKDIR /root -RUN apt-get update && apt-get install -y --no-install-recommends build-essential ca-certificates cmake git python3-pip python3-dev -# Cause torch CPU only to be installed, no cuda dependencies -RUN pip install torch --extra-index-url https://download.pytorch.org/whl/cpu -RUN git clone https://github.com/TorchDSP/torchsig -b v0.4.1 -WORKDIR /root/torchsig -# Remove dependencies that transforms don't need. -RUN sed -i -E "s/torch==[0-9\.]+/torch/g" pyproject.toml -RUN for d in h5py scikit-learn sympy timm torchmetrics torchvision pytorch_lightning ; do sed -i -E "/${d}/d" pyproject.toml ; done -RUN pip install . - -FROM iqtlabs/gamutrf:latest -WORKDIR /root -ENV DEBIAN_FRONTEND=noninteractive -# TODO: find a better way cherrypick just Torchsig itself, without Torch et al. Torchsig transforms have dependencies on -# Torch, even though we don't need Torch for the standalone transforms we want. -COPY --from=torchsig-builder /usr/local/lib/python3.12 /usr/local/lib/python3.12 -RUN python3 -c "from torchsig.transforms import transforms" -RUN python3 -c "from gamutrf import grscan" -COPY augment/augment.py /root/augment.py -# Ensure transforms work despite the dependency mangling. -RUN dd if=/dev/zero of=/tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000sps.raw bs=8 count=1000 && /root/augment.py /tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000sps.raw /tmp 1 "ST.Compose([ST.Identity()])" && diff -b /tmp/gamutrf_recording_ettus__gain40_1_1Hz_1000sps.raw /tmp/augmented-0-gamutrf_recording_ettus__gain40_1_1Hz_1000sps && rm -f /tmp/gamutrf* - -ENTRYPOINT ["/root/augment.py"]