Merge pull request #1046 from anarkiwi/wf

Move waterfall

.github/workflows/ci-test.yml

@@ -49,34 +49,37 @@ jobs:
           done && \
           sudo ldconfig -v && \
           poetry config virtualenvs.create false && \
-          cd gamutrflib && \
-          poetry install --no-interaction && \
-          cd .. && \
-          poetry install --no-interaction && \
+          poetry install --no-interaction -C gamutrflib && \
+          poetry install --no-interaction -C gamutrfwaterfall && \
           poetry install --no-interaction -C utils/mavlink-api && \
+          poetry install --no-interaction && \
           sudo dpkg -r --force-depends python3-numpy
       - name: Code Quality - Black
         run: |
+          poetry run black augment --check
           poetry run black gamutrf --check
-          poetry run black utils --check
+          poetry run black gamutrflib --check
+          poetry run black gamutrfwaterfall --check
           poetry run black torchserve --check
-          poetry run black augment --check
+          poetry run black utils --check
       - name: Code Quality - Pylint
         run: |
-          poetry run pylint --fail-under=6 gamutrf/
+          poetry run pylint --fail-under=6 gamutrf/ gamutrflib/ gamutrfwaterfall/
       - name: Code Quality - Pytype
         env:
           PYTHONPATH: /usr/local/lib/python3.10/dist-packages:/usr/lib/python3/dist-packages
         run: |
           sudo pip3 install pytype=="$(grep -E "pytype = " pyproject.toml | grep -Eo "[0-9\.]+")" && \
             sudo pip3 install -U pyserial && \
             pytype -k gamutrf/ && \
+            pytype -k gamutrflib/ && \
+            pytype -k gamutrfwaterfall/ && \
             pytype -k utils/mavlink-api
       - name: Test with pytest
         env:
           PYTHONPATH: /usr/local/lib/python3.10/dist-packages:/usr/lib/python3/dist-packages
         run: |
-          poetry run pytest --cov-report term-missing --cov=. --cov-report=xml tests/
+          poetry run pytest --cov-report term-missing --cov=. --cov-report=xml tests/ gamutrfwaterfall/tests
       - name: Upload coverage
         uses: codecov/codecov-action@v3
         with:

.github/workflows/docker-test.yml

@@ -33,13 +33,14 @@ jobs:
         docker build -f docker/Dockerfile.driver docker -t iqtlabs/gamutrf-driver:latest
         docker build -f docker/Dockerfile.uhd-sr docker -t iqtlabs/gamutrf-uhd-sr:latest
         docker build -f docker/Dockerfile.base docker -t iqtlabs/gamutrf-base:latest
+        docker build -f docker/Dockerfile.waterfall . -t iqtlabs/gamutrf-waterfall:latest
         docker build -f Dockerfile . -t iqtlabs/gamutrf:latest
         docker build -f docker/Dockerfile.torchsig . -t iqtlabs/gamutrf-torchsig:latest
         docker run -t iqtlabs/gamutrf:latest gamutrf-compress_dirs --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-offline --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-scan --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-sigfinder --help
-        docker run -t iqtlabs/gamutrf:latest gamutrf-waterfall --help
         docker run -t iqtlabs/gamutrf:latest gamutrf-worker --help
+        docker run -t iqtlabs/gamutrf-waterfall:latest gamutrf-waterfall --help
         sudo apt-get update && sudo apt-get install -qy python3-pip
         docker compose -f orchestrator.yml -f worker.yml -f docker/monitoring.yml -f specgram.yml build

.github/workflows/docker.yml

@@ -67,6 +67,15 @@ jobs:
         push: true
         tags: iqtlabs/gamutrf-base:${{ steps.change_version.outputs.VERSION }}
       if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
+    - name: Build and push platforms waterfall
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: docker/Dockerfile.waterfall
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: iqtlabs/gamutrf-waterfall:${{ steps.change_version.outputs.VERSION }}
+      if: github.repository == 'iqtlabs/gamutrf' && github.event_name == 'push'
   buildx-gamutrf:
     needs: [buildx-base]
     runs-on: ubuntu-latest

.gitignore

@@ -141,3 +141,6 @@ tests/gamutrf_test_dir/*.tar*
 *.DS_Store
 
 \#*
+torchserve/*mar
+torchserve/*txt
+torchserve/*pt

Dockerfile

@@ -14,9 +14,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y -q \
     libev-dev \
     python3 \
     python3-dev \
+    python3-pip \
     python3-pyqt5 \
-    python3-pyqt5.sip \
-    python3-pip && \
+    python3-pyqt5.sip && \
     curl -sSL https://install.python-poetry.org | python3 - --version 1.4.2 && \
     poetry config virtualenvs.create false && \
     python3 -m pip install --no-cache-dir --upgrade pip
@@ -67,6 +67,8 @@ RUN if [ "$(arch)" = "x86_64" ] ; then \
         libopencv-core4.5d \
         libopencv-imgcodecs4.5d \
         libopencv-imgproc4.5d \
+        python3-pyqt5 \
+        python3-pyqt5.sip \
         librtlsdr0 \
         libspdlog1 \
         libuhd4.1.0 \
@@ -75,10 +77,7 @@ RUN if [ "$(arch)" = "x86_64" ] ; then \
         libzmq5 \
         mesa-vulkan-drivers \
         python3 \
-        python3-pyqt5 \
-        python3-pyqt5.sip \
         python3-zmq \
-        sox \
         uhd-host \
         wget \
         zstd && \

docker/Dockerfile.waterfall

@@ -0,0 +1,39 @@
+# nosemgrep:github.workflows.config.dockerfile-source-not-pinned
+FROM ubuntu:22.04 as installer
+ENV DEBIAN_FRONTEND noninteractive
+ENV PATH="${PATH}:/root/.local/bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update && apt-get install --no-install-recommends -y -q \
+    ca-certificates \
+    curl \
+    gcc \
+    git \
+    libcairo2-dev \
+    libev-dev \
+    python3 \
+    python3-dev \
+    python3-pip && \
+    curl -sSL https://install.python-poetry.org | python3 - --version 1.4.2 && \
+    poetry config virtualenvs.create false
+WORKDIR /gamutrflib
+COPY gamutrflib /gamutrflib/
+RUN poetry install --no-interaction --no-ansi --no-dev
+WORKDIR /gamutrfwaterfall
+COPY gamutrfwaterfall /gamutrfwaterfall/
+RUN poetry install --no-interaction --no-ansi --no-dev
+
+FROM ubuntu:22.04
+ENV DEBIAN_FRONTEND noninteractive
+ENV PATH="${PATH}:/root/.local/bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-get update && apt-get install --no-install-recommends -y -q \
+    curl \
+    git \
+    libev4 \
+    libcairo2 \
+    python3
+COPY --from=installer /root/.local /root/.local
+COPY --from=installer /usr/local /usr/local
+COPY --from=installer /gamutrflib /gamutrflib
+COPY --from=installer /gamutrfwaterfall /gamutrfwaterfall
+CMD ["gamutrf-waterfall", "--help"]

gamutrf/__main__.py

@@ -4,7 +4,6 @@
 from gamutrf.scan import main as scan_main
 from gamutrf.sigfinder import main as sigfinder_main
 from gamutrf.specgram import main as specgram_main
-from gamutrf.waterfall import main as waterfall_main
 from gamutrf.worker import main as worker_main
 
 
@@ -33,11 +32,6 @@ def specgram():
     specgram_main()
 
 
-def waterfall():
-    """Entrypoint for waterfall"""
-    waterfall_main()
-
-
 def worker():
     """Entrypoint for worker"""
     worker_main()

gamutrf/waterfall.py → gamutrfwaterfall/gamutrfwaterfall/waterfall.py

@@ -22,13 +22,14 @@
 from scipy.ndimage import gaussian_filter
 
 from gamutrflib.peak_finder import get_peak_finder
-from gamutrf.utils import SCAN_FRES
 from gamutrflib.zmqbucket import ZmqReceiver, parse_scanners, frame_resample
 
 warnings.filterwarnings(action="ignore", message="Mean of empty slice")
 warnings.filterwarnings(action="ignore", message="All-NaN slice encountered")
 warnings.filterwarnings(action="ignore", message="Degrees of freedom <= 0 for slice.")
 
+SCAN_FRES = 1e4
+
 
 def safe_savefig(path):
     basename = os.path.basename(path)