From 607c6daf712ab3c0e44d71334b0d568ad1fde01f Mon Sep 17 00:00:00 2001 From: kx1t <15090643+kx1t@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:02:07 -0400 Subject: [PATCH] Renewed Autogain and fixes to HealthCheck (#80) * initial commit - new autogain * updates * updates * updates * update * update * updates * bug fix * reset stats on dump978 restart * make stats gen after restart smoother * cleanup * more changes * logging improvements * logging change * don't increment counter if stats aren't ready * Avoid error traps * Start counting runs from 1 instead of 0 * allow suspension of autogain outside certain hours * typo and other stuff * yet another typo :( * stupid errors fixed * stupid errors fixed * ensure directories are created * fix resume time * fix resume time * fix resume time * improve Dockerfile * improve flow * readme updates * made message type (raw v accepted) and count configurable * make `DUMP978_AUTOGAIN_ADJUSTMENT_LIMITS` default true * up READSB_AUTOGAIN_HIGH_PCT to 12.0 * simplify rssi data collection * updated autogain collection mechanism * updated autogain collection mechanism * updated autogain collection mechanism * updated autogain collection mechanism * updated autogain collection mechanism * updated autogain collection mechanism * updated autogain collection mechanism * more stuff * more stuff * more stuff * more stuff * more stuff * autogain updates * update autogain * remove some debug code * updates * update to flow * recognize `$READSB_AUTOGAIN_USE_RAW` parameter for autogain * always collect autogain msgs until at least READSB_AUTOGAIN_MIN_SAMPLES is reached * fix for RSSIs that are in the -0.9 - 0.0 range * respect `DUMP978_AUTOGAIN_INITIAL_GAIN` on startup with autogain * set agreed default values * multiple fixes for HealthCheck * Prep for future telegraf-less container * final cleanup before merging * linting "fix" --- .github/workflows/markdownlint.yml | 2 +- .markdownlint.json | 3 +- Dockerfile | 54 +---- README.md | 180 +++++++---------- .../dependencies.d/01-timezone | 0 .../dependencies.d/02-sanity-check | 0 .../dependencies.d/04-telegraf | 0 .../dependencies.d/05-rtlsdr-biastee | 0 .../s6-rc.d/06-initialise-gain/type | 1 - .../s6-overlay/s6-rc.d/06-initialise-gain/up | 2 - .../dependencies.d/06-initialise-gain | 0 .../dump978/dependencies.d/06-initialise-gain | 0 .../dependencies.d/06-initialise-gain | 0 .../readsb/dependencies.d/06-initialise-gain | 0 .../dependencies.d/06-initialise-gain | 0 .../s6-rc.d/stats/dependencies.d/01-timezone | 0 .../stats/dependencies.d/02-sanity-check | 0 .../s6-rc.d/stats/dependencies.d/04-telegraf | 0 .../stats/dependencies.d/05-rtlsdr-biastee | 0 .../stats/dependencies.d/06-initialise-gain | 0 .../s6-rc.d/stats/dependencies.d/telegraf | 0 rootfs/etc/s6-overlay/s6-rc.d/stats/run | 2 - rootfs/etc/s6-overlay/s6-rc.d/stats/type | 1 - .../dependencies.d/06-initialise-gain | 0 .../dependencies.d/06-initialise-gain | 0 .../user/contents.d/06-initialise-gain | 0 .../s6-overlay/s6-rc.d/user/contents.d/stats | 0 rootfs/etc/s6-overlay/scripts/02-sanity-check | 5 + rootfs/etc/s6-overlay/scripts/04-telegraf | 13 +- .../etc/s6-overlay/scripts/06-initialise-gain | 29 --- rootfs/etc/s6-overlay/scripts/autogain | 162 +++++++++++++-- rootfs/etc/s6-overlay/scripts/dump978 | 37 ++-- rootfs/etc/s6-overlay/scripts/readsb | 16 +- rootfs/etc/s6-overlay/scripts/skyaware978 | 17 +- rootfs/etc/s6-overlay/scripts/telegraf | 8 +- rootfs/etc/s6-overlay/scripts/telegraf_socat | 5 + rootfs/scripts/healthcheck.sh | 66 ++++--- rootfs/usr/local/bin/autogain978 | 187 ++++++++++++++++++ 38 files changed, 528 insertions(+), 262 deletions(-) delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/01-timezone delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/02-sanity-check delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/04-telegraf delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/05-rtlsdr-biastee delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/type delete mode 100755 rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/up delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/autogain/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/dump978/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/lighttpd/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/readsb/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/skyaware978/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/01-timezone delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/02-sanity-check delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/04-telegraf delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/05-rtlsdr-biastee delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/telegraf delete mode 100755 rootfs/etc/s6-overlay/s6-rc.d/stats/run delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/stats/type delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/telegraf/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/telegraf_socat/dependencies.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/06-initialise-gain delete mode 100644 rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/stats delete mode 100755 rootfs/etc/s6-overlay/scripts/06-initialise-gain create mode 100755 rootfs/usr/local/bin/autogain978 diff --git a/.github/workflows/markdownlint.yml b/.github/workflows/markdownlint.yml index 664d54e..37daea9 100644 --- a/.github/workflows/markdownlint.yml +++ b/.github/workflows/markdownlint.yml @@ -20,4 +20,4 @@ jobs: - name: Pull markdownlint/markdownlint:latest Image run: docker pull markdownlint/markdownlint:latest - name: Run markdownlint against *.md files - run: docker run --rm -i -v "$(pwd)":/workdir --workdir /workdir markdownlint/markdownlint:latest --rules ~MD004,~MD013,~MD033,~MD026,~MD002,~MD022 $(find . -type f -iname '*.md' | grep -v '/.git/') + run: docker run --rm -i -v "$(pwd)":/workdir --workdir /workdir markdownlint/markdownlint:latest --rules ~MD004,~MD013,~MD033,~MD026,~MD002,~MD022,~MD007,~MD029,~MD012,~MD034 $(find . -type f -iname '*.md' | grep -v '/.git/') diff --git a/.markdownlint.json b/.markdownlint.json index 79aa8d7..016214f 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -1,3 +1,4 @@ { - "MD033": false + "MD033": false, + "MD013": false } diff --git a/Dockerfile b/Dockerfile index 5817275..321e1b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,44 +15,7 @@ FROM ghcr.io/sdr-enthusiasts/docker-baseimage:dump978-full ENV PROMETHEUSPORT=9273 \ PROMETHEUSPATH="/metrics" \ - ########################################################################### - ##### AUTOGAIN ENVIRONMENT VARS ##### - # How often the autogain.sh is run (in seconds) - AUTOGAIN_SERVICE_PERIOD=900 \ - # The autogain state file (init/finetune/finish) - AUTOGAIN_STATE_FILE="/run/autogain/state" \ - # The current gain figure as-set by autogain - AUTOGAIN_CURRENT_VALUE_FILE="/run/autogain/autogain_current_value" \ - # The timestamp (seconds since epoch) when the current gain figure was set - AUTOGAIN_CURRENT_TIMESTAMP_FILE="/run/autogain/autogain_current_timestamp" \ - # The timestamp (seconds since epoch) when the current gain figure should be reviewed - AUTOGAIN_REVIEW_TIMESTAMP_FILE="/run/autogain/autogain_review_timestamp" \ - # The maximum allowable percentage of strong messages - AUTOGAIN_PERCENT_STRONG_MESSAGES_MAX=10.0 \ - # The minimum allowable percentage of strong messages - AUTOGAIN_PERCENT_STRONG_MESSAGES_MIN=0.5 \ - # The number of seconds that autogain "init" stage should run for, for each gain level - AUTOGAIN_INITIAL_PERIOD=7200 \ - # The minimum number of local_accepted messages that autogain "init" stage should run for, for each gain level - AUTOGAIN_INITIAL_MSGS_ACCEPTED=100000 \ - # The number of seconds that autogain "finetune" stage should run for, for each gain level - AUTOGAIN_FINETUNE_PERIOD=604800 \ - # The minimum number of local_accepted messages that autogain "finetune" stage should run for, for each gain level - AUTOGAIN_FINETUNE_MSGS_ACCEPTED=700000 \ - # How long to run once finetune stage has finished before we start the process over (1 year) - AUTOGAIN_FINISHED_PERIOD=31536000 \ - # Maximum gain level that autogain should use - AUTOGAIN_MAX_GAIN_VALUE=49.6 \ - # Minimum gain level that autogain should use - AUTOGAIN_MIN_GAIN_VALUE=0.0 \ - # State file that will disappear when the container is rebuilt/restarted - so autogain can detect container restart/rebuild - AUTOGAIN_RUNNING_FILE="/tmp/.autogain_running" \ - # maximum accepted gain value - AUTOGAIN_MAX_GAIN_VALUE_FILE="/run/autogain/autogain_max_value" \ - # minimum accepted gain value - AUTOGAIN_MIN_GAIN_VALUE_FILE="/run/autogain/autogain_min_value" \ - # Current gain value - GAIN_VALUE_FILE="/tmp/.gain_current" + GAIN_VALUE_FILE="/var/globe_history/autogain/gain" SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -95,17 +58,7 @@ RUN set -x && \ && \ # grab the bias t scripts curl -o /etc/s6-overlay/scripts/05-rtlsdr-biastee-init https://raw.githubusercontent.com/sdr-enthusiasts/sdre-bias-t-common/main/09-rtlsdr-biastee-init && \ - curl -o /etc/s6-overlay/scripts/05-rtlsdr-biastee-down https://raw.githubusercontent.com/sdr-enthusiasts/sdre-bias-t-common/main/09-rtlsdr-biastee-down && \ - # fix bias t init - sed -i 's/READSB_DEVICE_TYPE/DUMP978_DEVICE_TYPE/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-init && \ - sed -i 's/READSB_RTLSDR_DEVICE/DUMP978_RTLSDR_DEVICE/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-init && \ - sed -i 's/READSB_ENABLE_BIASTEE/DUMP978_ENABLE_BIASTEE/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-init && \ - sed -i 's/09-rtlsdr-biastee/05-rtlsdr-biastee/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-init && \ - # fix bias t down - sed -i 's/READSB_DEVICE_TYPE/DUMP978_DEVICE_TYPE/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-down && \ - sed -i 's/READSB_RTLSDR_DEVICE/DUMP978_RTLSDR_DEVICE/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-down && \ - sed -i 's/READSB_ENABLE_BIASTEE/DUMP978_ENABLE_BIASTEE/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-down && \ - sed -i 's/09-rtlsdr-biastee/05-rtlsdr-biastee/g' /etc/s6-overlay/scripts/05-rtlsdr-biastee-down && \ + curl -o /etc/s6-overlay/scripts/05-rtlsdr-biastee-down https://raw.githubusercontent.com/sdr-enthusiasts/sdre-bias-t-common/main/09-rtlsdr-biastee-down && \ chmod +x /etc/s6-overlay/scripts/05-rtlsdr-biastee-init && \ chmod +x /etc/s6-overlay/scripts/05-rtlsdr-biastee-down && \ git config --global advice.detachedHead false && \ @@ -153,6 +106,3 @@ EXPOSE 30978/tcp 30979/tcp 37981/tcp # Add healthcheck HEALTHCHECK --timeout=60s --start-period=7200s --interval=600s CMD /scripts/healthcheck.sh - -# TODO -# - work out a way to test - maybe capture some output and parse it? diff --git a/README.md b/README.md index ca10da1..e8de4d0 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,13 @@ - [`dump978-fa` General Options](#dump978-fa-general-options) - [`dump978-fa` RTL-SDR Options](#dump978-fa-rtl-sdr-options) - [General SDR Options](#general-sdr-options) - - [Auto-Gain Options](#auto-gain-options) - [InfluxDB Options](#influxdb-options) - [Prometheus Options](#prometheus-options) + - [Autogain Options](#autogain-options) + - [Autogain system](#autogain-system) + - [Forcing autogain to re-run from scratch](#forcing-autogain-to-re-run-from-scratch) + - [Container log messages while gain adjustments are made](#container-log-messages-while-gain-adjustments-are-made) - [`dump978` Web Pages](#dump978-web-pages) - - [Auto-Gain system](#auto-gain-system) - - [Initialisation Stage](#initialisation-stage) - - [Fine-Tuning Stage](#fine-tuning-stage) - - [Finished Stage](#finished-stage) - - [State/Log/Stats Files](#statelogstats-files) - - [Forcing auto-gain to re-run from scratch](#forcing-auto-gain-to-re-run-from-scratch) - [Logging](#logging) - [Getting help](#getting-help) @@ -51,9 +48,9 @@ The container listens on the following TCP ports: ## Paths & Volumes -| Path (inside container) | Details | -| ----------------------- | --------------------------------------------------------------------- | -| `/run/autogain` | Map this to persistent storage if you set `DUMP978_SDR_GAIN=autogain` | +| Path (inside container) | Details | +|-------------------------|---------| +| `/var/globe_history` | Map this to persistant storage if you set `DUMP978_SDR_GAIN=autogain` | ## Up and Running - `docker run` @@ -93,32 +90,31 @@ Here is an example `docker-compose.yml`: <‐‐ Click the arrow to see the docker-compose.yml text ```yaml -dump978: - # dump978 gets UAT data from the SDR - image: ghcr.io/sdr-enthusiasts/docker-dump978 - # profiles: - # - donotstart - tty: true - container_name: dump978 - hostname: dump978 - restart: always - labels: - - "autoheal=true" - device_cgroup_rules: - - "c 189:* rwm" - environment: - - TZ=${FEEDER_TZ} - - DUMP978_SDR_PPM=${ADSB_SDR_PPM} - - DUMP978_SDR_GAIN=${ADSB_SDR_GAIN} - - DUMP978_RTLSDR_DEVICE=${UAT_SDR_SERIAL} - - DUMP978_SDR_GAIN=${UAT_SDR_GAIN} - - DUMP978_SDR_PPM=${UAT_SDR_PPM} - - AUTOGAIN_INITIAL_PERIOD=7200 - volumes: - - /opt/adsb/dump978/autogain:/run/autogain - - /dev:/dev:ro - tmpfs: - - /run/readsb + dump978: +# dump978 gets UAT data from the SDR + image: ghcr.io/sdr-enthusiasts/docker-dump978 +# profiles: +# - donotstart + tty: true + container_name: dump978 + hostname: dump978 + restart: always + labels: + - "autoheal=true" + device_cgroup_rules: + - 'c 189:* rwm' + environment: + - TZ=${FEEDER_TZ} + - DUMP978_SDR_PPM=${ADSB_SDR_PPM} + - DUMP978_SDR_GAIN=${ADSB_SDR_GAIN} + - DUMP978_RTLSDR_DEVICE=${UAT_SDR_SERIAL} + - DUMP978_SDR_GAIN=${UAT_SDR_GAIN} + - DUMP978_SDR_PPM=${UAT_SDR_PPM} + volumes: + - /opt/adsb/dump978:/var/globe_history + - /dev:/dev:ro + tmpfs: + - /run ultrafeeder: image: ghcr.io/sdr-enthusiasts/docker-adsb-ultrafeeder @@ -282,13 +278,13 @@ You should now be feeding ADSB-ES & UAT to the "new" aggregators, FlightAware, a Where the default value is "Unset", `dump978-fa`'s default will be used. -| Variable | Description | Controls which `dump978-fa` option | Default | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | -------- | -| `DUMP978_DEVICE_TYPE` | Currently only `rtlsdr` is supported. If you have another type of radio, please open an issue and I'll try to get it added. | `--sdr driver=` | `rtlsdr` | -| `DUMP978_SDR_AGC` | Set to any value to enable SDR AGC. | `--sdr-auto-gain` | Unset | -| `DUMP978_SDR_GAIN` | Set gain (in dB). Use autogain to have the container determine an appropriate gain, more on this below. | `--sdr-gain` | Unset | -| `DUMP978_SDR_PPM` | Set SDR frequency correction in PPM. | `--sdr-ppm` | Unset | -| `DUMP978_JSON_STDOUT` | Write decoded json to the container log. Useful for troubleshooting, but don't leave enabled! | `--json-stdout` | Unset | +| Variable | Description | Controls which `dump978-fa` option | Default | +|----------|-------------|--------------------------------|---------| +| `DUMP978_DEVICE_TYPE` | Currently only `rtlsdr` is supported. If you have another type of radio, please open an issue and I'll try to get it added. | `--sdr driver=` | `rtlsdr` | +| `DUMP978_SDR_AGC` | Set to any value to enable SDR AGC. | `--sdr-auto-gain` | Unset | +| `DUMP978_SDR_GAIN` | Set gain (in dB). Use autogain to have the container determine an appropriate gain, more on this below. | `--sdr-gain` | Unset | +| `DUMP978_SDR_PPM` | Set SDR frequency correction in PPM. | `--sdr-ppm` | Unset | +| `DUMP978_JSON_STDOUT` | Write decoded json to the container log. Useful for troubleshooting, but don't leave enabled! | `--json-stdout` | Unset | ### `dump978-fa` RTL-SDR Options @@ -306,23 +302,6 @@ Where the default value is "Unset", `dump978-fa`'s default will be used. | ------------------------ | ---------------------------------------------------- | ------- | | `DUMP978_ENABLE_BIASTEE` | Set to any value to enable bias-tee on your RTL-SDR. | Unset | -### Auto-Gain Options - -These variables control the auto-gain system (explained further below). These should rarely need changing from the defaults. - -| Variable | Description | Default | -| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | -| `AUTOGAIN_INITIAL_PERIOD` | How long each gain level should be measured during auto-gain initialisation (ie: "roughing in"), in seconds. | `7200` (2 hours) | -| `AUTOGAIN_INITIAL_MSGS_ACCEPTED` | How many locally accepted messages should be received per gain level during auto-gain initialisaion to ensure accurate measurement. | `100000` | -| `AUTOGAIN_FINETUNE_PERIOD` | How long each gain level should be measured during auto-gain fine-tuning, in seconds. | `604800` (7 days) | -| `AUTOGAIN_FINETUNE_MSGS_ACCEPTED` | How many locally accepted messages should be received per gain level during auto-gain fine-tuning to ensure accurate measurement. | `700000` | -| `AUTOGAIN_FINISHED_PERIOD` | How long between the completion of fine-tuning (and ultimately setting a preferred gain), and re-running the entire process. | `31536000` (1 year) | -| `AUTOGAIN_MAX_GAIN_VALUE` | The maximum gain setting in dB that will be used by auto-gain. | `49.6` (max supported by `readsb`) | -| `AUTOGAIN_MIN_GAIN_VALUE` | The minimum gain setting in dB that will be used by auto-gain. | `0.0` (min supported by `readsb`) | -| `AUTOGAIN_PERCENT_STRONG_MESSAGES_MAX` | The maximum percentage of "strong messages" auto-gain will aim for. | `10.0` | -| `AUTOGAIN_PERCENT_STRONG_MESSAGES_MIN` | The minimum percentage of "strong messages" auto-gain will aim for. | `0.5` | -| `AUTOGAIN_SERVICE_PERIOD` | How often the auto-gain system will check results and perform actions, in seconds | `900` (15 minutes) | - ### InfluxDB Options These variables control the sending of flight data and dump978 metrics to [InfluxDB](https://docs.influxdata.com/influxdb/) (via a built-in instance of [Telegraf](https://docs.influxdata.com/telegraf/)). @@ -348,72 +327,49 @@ These variables control exposing flight data to [Prometheus](https://prometheus. | `PROMETHEUSPORT` | The port that the prometheus client will listen on | `9273` | | `PROMETHEUSPATH` | The path that the prometheus client will publish metrics on | `/metrics` | -## `dump978` Web Pages +### Autogain Options -The container's webserver makes SkyAware978 (and the related `data` directories with `json` statistics files) available at `/skyaware978`. This means, using the port mapping example shown above, that you can access these URLs (among other things): +These variables control the autogain system (explained further below). These should rarely need changing from the defaults. -- [http://my_ip:30980/skyaware978](http://my_ip:30980/skyaware978) -- SkyAware978 map -- [http://my_ip:30980/skyaware978/data/aircraft.json](http://my_ip:30980/skyaware978/data/aircraft.json) -- aircraft.json statistics file -- [http://my_ip:30980/health](http://my_ip:30980/health) -- HealthCheck results (returns `OK` if container is healthy) +| Variable | Description | Default | +|----------|-------------|---------| +| `DUMP978_AUTOGAIN_INITIAL_TIMEPERIOD` | How long the autogain initialization phase should take (ie: "roughing in"), in seconds. | `21600` (6 hours) | +| `DUMP978_AUTOGAIN_INITIAL_INTERVAL` | How often autogain should measure and adjust the gain during the initialization phase, in seconds. | `600` (10 minutes) | +| `DUMP978_AUTOGAIN_SUBSEQUENT_INTERVAL` | How often autogain should measure and adjust the gain after the initialization phase is done, in seconds. | `84600` (24 hours) | +| `DUMP978_AUTOGAIN_ADJUSTMENT_LIMITS` | If set to `true`/`on`/`yes`/`1`, while in the initialization phase, autogain will only adjust the gain during the timeframe set by the `DUMP978_AUTOGAIN_ADJUSTMENT_TIMEFRAME` parameter. | `true` | +| `DUMP978_AUTOGAIN_ADJUSTMENT_TIMEFRAME` | Timeframe limits for autogain during the initializaion phase, in `HHMM-HHMM` (start hours/minutes to end hours/minutes). If an adjustment "run" falls outside these limits, the autogain adjustment is delayed until the start of the next timeframe. Times are based on the container's Timezone (`TZ`) setting. | `0900-1800` (9 AM - 6 PM, local container time) | +| `DUMP978_AUTOGAIN_LOW_PCT` | If the percentage of "strong signals" (>3dB) over a measuring period is less than this parameter, the gain will be increased by 1 position | `2.5` (2.5%) | +| `DUMP978_AUTOGAIN_HIGH_PCT` | If the percentage of "strong signals" (>3dB) over a measuring period is more than this parameter, the gain will be decreased by 1 position | `6.0` (6.0%) | +| `READSB_AUTOGAIN_MIN_SAMPLES` | Minimum number of received samples for autogain to be able to consider adjusting the gain | `1000` | +| `READSB_AUTOGAIN_USE_RAW` | If set to `true`/`on`/`yes`/`1`, the autogain function will use the "raw" message count rather than the "accepted" message count. | `true` | -## Auto-Gain system +## Autogain system -An automatic gain adjustment system is included in this container, and can be activated by setting the environment variable `DUMP978_SDR_GAIN` to `autogain`. You should also map `/run/autogain` to persistent storage, otherwise the auto-gain system will start over each time the container is restarted. You should also ensure `LAT` and `LON` are set because the script uses the maximum range as a metric for choosing the best gain level. +An automatic gain adjustment system is included in this container, and can be activated by setting the environment variable `DUMP978_SDR_GAIN` to `autogain`. You should also map `/var/globe_history/` to persistent storage, otherwise the autogain system will start over each time the container is restarted. -_Why is this written in bash?_ Because I wanted to keep the container size down and not have to install an interpreter like python. I don't know C/Go/Perl or any other languages. +Autogain will take several hours to initially work out a reasonable gain. This is the so-called "initialization period", which is by default 6 hours. It will then perform a daily measurement to see if your gain needs further adjusting. -Auto-gain will take several weeks to initially (over the period of a week or so) work out feasible maximum and minimum gain levels for your environment. It will then perform a fine-tune process to find the optimal gain level. +The autogain system will work as follows; values are based on the default parameter settings from above: -During each process, gain levels are ranked as follows: +1. `dump978` is set to maximum gain. +2. Initial results are collected every 10 minutes, for up to 6 hours (initialization phase). If `DUMP978_AUTOGAIN_ADJUSTMENT_LIMITS` is set to true, measurements are suspended if the time is outside the set time limits (0900 - 1800 local container time). Every 10 minutes, the gain is adjusted by 1 position if the average percentage of "strong" signals (>-3dB) is less than 2.5% or more than 6.0%. +3. After the initialization phase is over, the average percentage of "strong signal" is calculated on a daily basis, and an adjustment is made accordingly. -- The range achievable by each gain level -- The signal-to-noise ratio of the receiver +### Forcing autogain to re-run from scratch -The ranking process is done by sorting the gain levels for each statistic from worst to best, then awarding points. 0 points are awarded for the worst gain level, 1 point for the next gain level all the way up to several points for the best gain level (total number of points is the number of gain levels tested). The number of points for each gain level is totalled, and the optimal gain level is the level with the largest number of points. Any gain level with a percentage of "strong signals" outside of `AUTOGAIN_PERCENT_STRONG_MESSAGES_MAX` and `AUTOGAIN_PERCENT_STRONG_MESSAGES_MIN` is discarded. +Run `docker exec dump978 autogain978 reset` to remove reset all autogain data and start the initialization phase fron scratch -Using this method, auto-gain tried to achieve the best balance of range, tracks and signal-to-noise ratio, whilst ensuring an appropriate number of "strong signals". +### Container log messages while gain adjustments are made -The auto-gain system will work as follows: +When a gain adjustment is made, `dump978-fa` and related processes are forcibly restarted. This will cause a number of messages to the container logs showing that these processes are terminated, and subsequently restarted. These messages are normal during an autogain gain adjustment, and are not errors in the container. -### Initialisation Stage - -In the initialisation process: - -1. `dump978` is set to maximum gain (`AUTOGAIN_MAX_GAIN_VALUE`). -1. Results are collected up to `AUTOGAIN_INITIAL_PERIOD` (up to 2 hours by default). -1. Check to ensure at least `AUTOGAIN_INITIAL_MSGS_ACCEPTED` messages have been locally accepted (1,000,000 by default). If not, continue collecting data for up to 24 hours. This combination of time and number of messages ensures we have enough data to make a valid initial assessment of each gain level. -1. Gain level is lowered by one level. -1. Gain levels are reviewed from lowest to highest gain level. If there have been gain levels resulting in a percentage of strong messages between `AUTOGAIN_PERCENT_STRONG_MESSAGES_MAX` and `AUTOGAIN_PERCENT_STRONG_MESSAGES_MIN`, and there have been three consecutive gain levels above `AUTOGAIN_PERCENT_STRONG_MESSAGES_MAX`, auto-gain lowers the maximum gain level. -1. Gain levels are reviewed from highest to lowest gain level. If there have been gain levels resulting in a percentage of strong messages between `AUTOGAIN_PERCENT_STRONG_MESSAGES_MAX` and `AUTOGAIN_PERCENT_STRONG_MESSAGES_MIN`, and there have been three consecutive gain levels below `AUTOGAIN_PERCENT_STRONG_MESSAGES_MIN`, auto-gain discontinues testing gain levels. - -Auto-gain then moves onto the fine-tuning stage. - -### Fine-Tuning Stage - -In the fine-tuning process: - -1. `dump978` is set to maximum gain level chosen at the end of the initialisation process. -1. Results are collected up to `AUTOGAIN_FINETUNE_PERIOD` (7 days by default). -1. Check to ensure at least `AUTOGAIN_FINETUNE_MSGS_ACCEPTED` messages have been locally accepted (7,000,000 by default). If not, continue collecting data for up to 48 hours. This combination of time and number of messages ensures we have enough data to make an accurate assessment of each gain level, and by using 7 days this ensures any peaks/troughs in data due to quiet/busy days of the week do not skew results. -1. Gain level is lowered by one level until the minimum gain level chosen at the end of the initialisation process is reached. - -At this point, all of the tested gain levels are ranked based on the criterea discussed above. - -The gain level with the most points is chosen, and `dump978` is set to this gain level. - -Auto-gain then moves onto the finished stage. - -### Finished Stage - -In the finished stage, auto-gain does nothing (as `dump978` is operating at optimal gain) for `AUTOGAIN_FINISHED_PERIOD` (1 year by default). After this time, auto-gain reverts to the initialisation stage and the entire process is completed again. This makes sure your configuration is always running at the optimal gain level as your RTLSDR ages. - -### State/Log/Stats Files - -All files for auto-gain are located at `/run/autogain` within the container. They should not be modified by hand. +## `dump978` Web Pages -### Forcing auto-gain to re-run from scratch +The container's webserver makes SkyAware978 (and the related `data` directories with `json` statistics files) available at `/skyaware978`. This means, using the port mapping example shown above, that you can access these URLs (among other things): -Run `docker exec rm /run/autogain/*` to remove all existing auto-gain state data. Restart the container and auto-gain will detect this and re-start at initialisation stage. +- [http://my_ip:30980/skyaware978](http://my_ip:30980/skyaware978) -- SkyAware978 map +- [http://my_ip:30980/skyaware978/data/aircraft.json](http://my_ip:30980/skyaware978/data/aircraft.json) -- aircraft.json statistics file +- [http://my_ip:30980/health](http://my_ip:30980/health) -- HealthCheck results (returns `OK` if container is healthy) ## Logging diff --git a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/01-timezone b/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/01-timezone deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/02-sanity-check b/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/02-sanity-check deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/04-telegraf b/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/04-telegraf deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/05-rtlsdr-biastee b/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/dependencies.d/05-rtlsdr-biastee deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/type b/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/type deleted file mode 100644 index bdd22a1..0000000 --- a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/type +++ /dev/null @@ -1 +0,0 @@ -oneshot diff --git a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/up b/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/up deleted file mode 100755 index cbd1bff..0000000 --- a/rootfs/etc/s6-overlay/s6-rc.d/06-initialise-gain/up +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec /etc/s6-overlay/scripts/06-initialise-gain diff --git a/rootfs/etc/s6-overlay/s6-rc.d/autogain/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/autogain/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/dump978/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/dump978/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/lighttpd/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/lighttpd/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/readsb/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/readsb/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/skyaware978/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/skyaware978/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/01-timezone b/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/01-timezone deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/02-sanity-check b/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/02-sanity-check deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/04-telegraf b/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/04-telegraf deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/05-rtlsdr-biastee b/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/05-rtlsdr-biastee deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/telegraf b/rootfs/etc/s6-overlay/s6-rc.d/stats/dependencies.d/telegraf deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/run b/rootfs/etc/s6-overlay/s6-rc.d/stats/run deleted file mode 100755 index fd5c86e..0000000 --- a/rootfs/etc/s6-overlay/s6-rc.d/stats/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec /etc/s6-overlay/scripts/stats diff --git a/rootfs/etc/s6-overlay/s6-rc.d/stats/type b/rootfs/etc/s6-overlay/s6-rc.d/stats/type deleted file mode 100644 index 5883cff..0000000 --- a/rootfs/etc/s6-overlay/s6-rc.d/stats/type +++ /dev/null @@ -1 +0,0 @@ -longrun diff --git a/rootfs/etc/s6-overlay/s6-rc.d/telegraf/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/telegraf/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/telegraf_socat/dependencies.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/telegraf_socat/dependencies.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/06-initialise-gain b/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/06-initialise-gain deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/stats b/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/stats deleted file mode 100644 index e69de29..0000000 diff --git a/rootfs/etc/s6-overlay/scripts/02-sanity-check b/rootfs/etc/s6-overlay/scripts/02-sanity-check index 6ce6b5f..d5419ca 100755 --- a/rootfs/etc/s6-overlay/scripts/02-sanity-check +++ b/rootfs/etc/s6-overlay/scripts/02-sanity-check @@ -36,4 +36,9 @@ if [[ ! -f /usr/local/bin/viewadsb ]]; then chmod a+x /usr/local/bin/viewadsb fi +# make sure directories exist +mkdir -p /run/skyaware978 +mkdir -p /run/stats +#mkdir -p /run/autogain + exit $EXITCODE diff --git a/rootfs/etc/s6-overlay/scripts/04-telegraf b/rootfs/etc/s6-overlay/scripts/04-telegraf index ee1231c..01aa06a 100755 --- a/rootfs/etc/s6-overlay/scripts/04-telegraf +++ b/rootfs/etc/s6-overlay/scripts/04-telegraf @@ -3,17 +3,17 @@ # Initialise config files, and remove existing OUTPUT_INFLUXDB_CONFIG_FILE="/etc/telegraf/telegraf.d/outputs_influxdb.conf" -rm "$OUTPUT_INFLUXDB_CONFIG_FILE" > /dev/null 2>&1 || true +rm -f "$OUTPUT_INFLUXDB_CONFIG_FILE" > /dev/null 2>&1 || true OUTPUT_PROMETHEUS_CONFIG_FILE="/etc/telegraf/telegraf.d/outputs_prometheus.conf" -rm "$OUTPUT_PROMETHEUS_CONFIG_FILE" > /dev/null 2>&1 || true +rm -f "$OUTPUT_PROMETHEUS_CONFIG_FILE" > /dev/null 2>&1 || true INPUT_JSON_CONFIG_FILE="/etc/telegraf/telegraf.d/inputs_socket_listener_dump978_json.conf" -rm "$INPUT_JSON_CONFIG_FILE" > /dev/null 2>&1 || true +rm -f "$INPUT_JSON_CONFIG_FILE" > /dev/null 2>&1 || true INPUT_POLAR_RANGE_DATA="/etc/telegraf/telegraf.d/inputs_polar_range.conf" -rm "$INPUT_POLAR_RANGE_DATA" > /dev/null 2>&1 || true +rm -f "$INPUT_POLAR_RANGE_DATA" > /dev/null 2>&1 || true INPUT_STATS_DATA="/etc/telegraf/telegraf.d/inputs_stats.conf" -rm "$INPUT_STATS_DATA" > /dev/null 2>&1 || true +rm -f "$INPUT_STATS_DATA" > /dev/null 2>&1 || true INPUT_AUTOGAIN_FILE="/etc/telegraf/telegraf.d/inputs_file_autogain.conf" -rm "$INPUT_AUTOGAIN_FILE" > /dev/null 2>&1 || true +rm -f "$INPUT_AUTOGAIN_FILE" > /dev/null 2>&1 || true if [[ -n "$ENABLE_PROMETHEUS" ]]; then @@ -208,5 +208,6 @@ if [[ -z "$INFLUXDB_SKIP_AIRCRAFT" ]]; then } > "$INPUT_JSON_CONFIG_FILE" fi +mkdir -p /run/stats touch /run/stats/stats.json touch /run/stats/polar_range.influx diff --git a/rootfs/etc/s6-overlay/scripts/06-initialise-gain b/rootfs/etc/s6-overlay/scripts/06-initialise-gain deleted file mode 100755 index 58ba821..0000000 --- a/rootfs/etc/s6-overlay/scripts/06-initialise-gain +++ /dev/null @@ -1,29 +0,0 @@ -#!/command/with-contenv bash -# shellcheck shell=bash - -# How to check if autogain needs to be restarted at current state/gain -rm "$AUTOGAIN_RUNNING_FILE" > /dev/null 2>&1 || true - -# If gain is specified... -if [[ -n "$DUMP978_SDR_GAIN" ]]; then - - # If the user wants to use the autogain system... - if [[ "$DUMP978_SDR_GAIN" == "autogain" ]]; then - - # Do we have a previously-stored gain value? - if [[ ! -e "$AUTOGAIN_CURRENT_VALUE_FILE" ]]; then - # if not, then set the gain file with the maximum gain - echo "49.6" > "$AUTOGAIN_CURRENT_VALUE_FILE" - fi - - # set the gain file with the previous autogain file - cp "$AUTOGAIN_CURRENT_VALUE_FILE" "$GAIN_VALUE_FILE" - - # If the user does not want to use the autogain system and has specified an actual gain figure, then use it - else - echo "$DUMP978_SDR_GAIN" > "$GAIN_VALUE_FILE" - fi -else - # Otherwise, use default of 49.6 (ie: maximum) - echo "49.6" > "$GAIN_VALUE_FILE" -fi diff --git a/rootfs/etc/s6-overlay/scripts/autogain b/rootfs/etc/s6-overlay/scripts/autogain index 3b007d0..cf71e35 100755 --- a/rootfs/etc/s6-overlay/scripts/autogain +++ b/rootfs/etc/s6-overlay/scripts/autogain @@ -1,22 +1,158 @@ #!/command/with-contenv bash -#shellcheck shell=bash +# shellcheck shell=bash disable=SC1091,SC2076,SC2154,SC2015 -# If gain is specified... -if [[ -n "$DUMP978_SDR_GAIN" ]]; then +source /scripts/common - # If the user wants to use the autogain system... - if [[ "$DUMP978_SDR_GAIN" == "autogain" ]]; then +trap 'pkill -P $$ || true; exit 0' SIGTERM SIGINT SIGHUP SIGQUIT +# set -x +# Autogain routine +# +# Relevant env variables: +# DUMP978_GAIN or READSB_GAIN: set to "autogain" to enable autogain +# DUMP978_AUTOGAIN_INITIAL_INTERVAL or READSB_AUTOGAIN_INITIAL_INTERVAL: time in seconds to run autogain during initial assessment period; 600 (=10 minutes) +# DUMP978_AUTOGAIN_SUBSEQUENT_INTERVAL or READSB_AUTOGAIN_SUBSEQUENT_INTERVAL: time in seconds to run autogain during initial assessment period; 86400 (=1 day) +# DUMP978_AUTOGAIN_INITIAL_TIMEPERIOD or READSB_AUTOGAIN_INITIAL_TIMEPERIOD: time in seconds that the initial gain assessment will last. Default 14400 (=4 hours) +# Command to restart autogain: docker exec -it dump978 /usr/local/bin/autogain978 reset - set -eo pipefail +# make READSB_AUTOGAIN.... the same as DUMP978_AUTOGAIN with the latter as preferred parameter +READSB_AUTOGAIN_INITIAL_INTERVAL="${DUMP978_AUTOGAIN_INITIAL_INTERVAL:-${READSB_AUTOGAIN_INITIAL_INTERVAL:-900}}" +READSB_AUTOGAIN_SUBSEQUENT_INTERVAL="${DUMP978_AUTOGAIN_SUBSEQUENT_INTERVAL:-${READSB_AUTOGAIN_SUBSEQUENT_INTERVAL:-86400}}" +READSB_AUTOGAIN_INITIAL_TIMEPERIOD="${DUMP978_AUTOGAIN_INITIAL_TIMEPERIOD:-${READSB_AUTOGAIN_INITIAL_TIMEPERIOD:-32400}}" +READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME="${DUMP978_AUTOGAIN_ADJUSTMENT_TIMEFRAME:-${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME:-0800-1800}}" +READSB_SDR_GAIN="${DUMP978_SDR_GAIN:-${READSB_SDR_GAIN:-autogain}}" +READSB_AUTOGAIN_ADJUSTMENT_LIMITS="${DUMP978_AUTOGAIN_ADJUSTMENT_LIMITS:-${READSB_AUTOGAIN_ADJUSTMENT_LIMITS:-true}}" +READSB_AUTOGAIN_STRONGSIGNAL_LIMIT="${DUMP978_AUTOGAIN_STRONGSIGNAL_LIMIT:-${READSB_AUTOGAIN_STRONGSIGNAL_LIMIT:--3.5}}" +READSB_AUTOGAIN_USE_RAW="${DUMP978_AUTOGAIN_USE_RAW:-${READSB_AUTOGAIN_USE_RAW:-true}}" +READSB_AUTOGAIN_MIN_SAMPLES="${DUMP978_AUTOGAIN_MIN_SAMPLES:-${READSB_AUTOGAIN_MIN_SAMPLES:-1000}}" - # Run autogain - #shellcheck disable=SC2016 - s6wrap --quiet --timestamps --prepend="$(basename "$0")" --args bash /scripts/autogain.sh - # Sleep for 15 mins - sleep "$AUTOGAIN_SERVICE_PERIOD" +function collect_gain_values() { + # reads RAW messages for $1 seconds and returns the percentage strong messages with 1 decimal precision + # $2 is optionally the cut-off RSSI value for strong signals + local total_msg=0 + local strong_msg=0 + local rssi + local endtime + local cutoff="${2:--3.5}" + cutoff="$(bc -l <<< "scale=0; ${2/#-} * 10 / 1")" + if [[ -z "$1" ]]; then + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "error - collect_gain_values function needs collection time as argument" + return 99 fi -else - sleep infinity + # wait until dump978-fa is running; autogain should wait for this process to exist + if ! pgrep dump978-fa >/dev/null 2>&1; then + while ! pgrep dump978-fa >/dev/null 2>&1; do + sleep 1 + done + sleep 5 # wait a few secs after dump978-fa comes up to allow it to get established + fi + + # make sure that any previous FD 3's are closed before we start. If they are still open when we + # (re)create the FD, the starting pointer for reading will be where we last left off reading, i.e. old data + exec 3>&- + # create a FD with a data stream of rssi values + if chk_enabled "$READSB_AUTOGAIN_USE_RAW"; then + # parse all messages that have an ";rssi=-xx.xx" element, regardless of any error indicators + while ! exec 3< <(stdbuf -oL sed -n 's|^.*;rssi=-\([0-9]\+\)\.\([0-9]*\);.*$|\1\2|p' /dev/null) 2>/dev/null; do + sleep 5 + exec 3>&- + done + else + # ignore messages that have anything between the raw message string and ";rssi=" + # These messages have an error code ";ts=x" between the raw msg and the rssi element + # In that case, we're only considering "accepted" messages + while ! exec 3< <(stdbuf -oL sed -n 's|^-[0-9a-f]*;rssi=-\([0-9]\+\)\.\([0-9]*\);.*$|\1\2|p' /dev/null) 2>/dev/null; do + sleep 5 + exec 3>&- + done + fi + + # read values for the collection time + starttime="$(date +%s)" + while IFS= read -r rssi <&3; do + (( total_msg++ )) || true + (( ${rssi##0} < cutoff )) && (( strong_msg++ )) || true + # read messages until we have least READSB_AUTOGAIN_MIN_SAMPLES, and the max time is exhausted + # in other words - even if the max time expires, continue collecting messages until at least the minimum has been reached + if (( $(date +%s) > starttime + $1 )) && (( total_msg > READSB_AUTOGAIN_MIN_SAMPLES )); then + break + fi + done + # close file descriptor + exec 3>&- + + echo -n "$total_msg;$strong_msg;" + if (( strong_msg * total_msg > 0 )); then + bc -l <<< "scale=1; 100 * $strong_msg / $total_msg / 1" + else + echo "na" + fi +} + +if [[ "${READSB_SDR_GAIN,,}" != "autogain" ]]; then + # Autogain is not enabled, so let's do nothing forever + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "Gain is set to ${READSB_SDR_GAIN}; autogain disabled." + exec sleep infinity fi + +mkdir -p /var/globe_history/autogain + +# Do special things if it's the first time AUTOGAIN is running +if [[ ! -f /var/globe_history/autogain/autogain_initialized ]]; then + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "Autogain initialization started. We'll collect data every $READSB_AUTOGAIN_INITIAL_INTERVAL secs for $(( READSB_AUTOGAIN_INITIAL_TIMEPERIOD / 60 )) minutes to do an initial gain assessment." + + # run autogain every $READSB_AUTOGAIN_INITIAL_INTERVAL secs for $READSB_AUTOGAIN_INITIAL_TIMEPERIOD secs + for (( i=1; i<=$(( READSB_AUTOGAIN_INITIAL_TIMEPERIOD / READSB_AUTOGAIN_INITIAL_INTERVAL )); i++ )) + do + # check if adjustment time limits are enabled and if we are in the window; if we're not then sleep until we will be in the window + if chk_enabled "$READSB_AUTOGAIN_ADJUSTMENT_LIMITS" && \ + (( $(date +%s) < $(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME%%-*} today" +%s) )); then + # current time is before start of window; delay until window starts + now_in_secs="$(date +%s)" + resumetime_in_secs="$(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME%%-*} today" +%s)" + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "Too early: current time before ${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME} adjustment time window. We will delay until $(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME%%-*}"), in $((resumetime_in_secs - now_in_secs))s" + sleep $((resumetime_in_secs - now_in_secs)) & wait $! + elif chk_enabled "$READSB_AUTOGAIN_ADJUSTMENT_LIMITS" && \ + (( $(date +%s) > $(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME##*-} today" +%s) )); then + # current time is after close of window; delay until window starts tomorrow + now_in_secs="$(date +%s)" + resumetime_in_secs="$(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME%%-*} today" +%s)" + if (( resumetime_in_secs < now_in_secs )); then resumetime_in_secs="$(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME%%-*} tomorrow" +%s)"; fi + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "Too late: current time outside of ${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME} adjustment time window. We will delay until $(date -d "${READSB_AUTOGAIN_ADJUSTMENT_TIMEFRAME%%-*} tomorrow"), in $((resumetime_in_secs - now_in_secs))s" + sleep "$((resumetime_in_secs - now_in_secs))s" & wait $! + fi + + # now collect data for $READSB_AUTOGAIN_INITIAL_INTERVAL seconds: + endtime="$(( $(date +%s) + READSB_AUTOGAIN_INITIAL_INTERVAL ))" + msg_stats="$(collect_gain_values "$READSB_AUTOGAIN_INITIAL_INTERVAL" "$READSB_AUTOGAIN_STRONGSIGNAL_LIMIT")" + if (( $(date +%s) < endtime )); then + # data collection exited early. Wait out the rest of the time and invalidate partial results +# s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "early exit after $(( READSB_AUTOGAIN_INITIAL_INTERVAL + $(date +%s) - endtime )) seconds" +# sleep "$(( endtime - $(date +%s) ))" + msg_stats="0;0;na" + fi + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "Autogain initialization run $i of $(( READSB_AUTOGAIN_INITIAL_TIMEPERIOD / READSB_AUTOGAIN_INITIAL_INTERVAL )) ($msg_stats)..." + if [[ $msg_stats == "0;0;na" ]] || ! s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args /usr/local/bin/autogain978 "$msg_stats"; then + ((i--)) || true + fi + + # sleep a little bit so dump978 is again providing data + sleep 15 + + done + touch /var/globe_history/autogain/autogain_initialized +fi + +s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args echo "Autogain long-term maintenance started. We'll collect data and assess every $(( READSB_AUTOGAIN_SUBSEQUENT_INTERVAL / 60 )) minutes if gain needs to be adjusted." +while true +do + endtime="$(( $(date +%s) + READSB_AUTOGAIN_SUBSEQUENT_INTERVAL ))" + msg_stats="$(collect_gain_values "$READSB_AUTOGAIN_SUBSEQUENT_INTERVAL" "$READSB_AUTOGAIN_STRONGSIGNAL_LIMIT")" + if (( $(date +%s) >= endtime )); then + s6wrap --quiet --prepend="$(basename "$0")" --timestamps --args /usr/local/bin/autogain978 "$msg_stats" + else + # data collection exited early. Wait a bit and restart + sleep 60 & wait ! + fi +done diff --git a/rootfs/etc/s6-overlay/scripts/dump978 b/rootfs/etc/s6-overlay/scripts/dump978 index 88d2d16..79c39af 100755 --- a/rootfs/etc/s6-overlay/scripts/dump978 +++ b/rootfs/etc/s6-overlay/scripts/dump978 @@ -7,26 +7,26 @@ s6wrap=(s6wrap --quiet --timestamps --prepend="$(basename "$0")" --args) DUMP978_BIN="/usr/local/bin/dump978-fa" # Listen on 0.0.0.0:DUMP978_RAW_PORT for raw messages -DUMP978_CMD=("--raw-port" "0.0.0.0:30978") +DUMP978_CMD=("--raw-port 0.0.0.0:30978") # Listen on 0.0.0.0:DUMP978_JSON_PORT for json messages -DUMP978_CMD+=("--json-port" "0.0.0.0:30979") +DUMP978_CMD+=("--json-port 0.0.0.0:30979") -DUMP978_CMD+=("--format" "CS8") +DUMP978_CMD+=("--format CS8") # Handle "--sdr" if [[ -n "${DUMP978_DEVICE_TYPE:-rtlsdr}" ]]; then case "${DUMP978_DEVICE_TYPE:-rtlsdr}" in rtlsdr) if [[ -n "${DUMP978_RTLSDR_DEVICE}" ]]; then - DUMP978_CMD+=("--sdr" "driver=rtlsdr,serial=${DUMP978_RTLSDR_DEVICE}") + DUMP978_CMD+=("--sdr driver=rtlsdr,serial=${DUMP978_RTLSDR_DEVICE}") else - DUMP978_CMD+=("--sdr" "driver=rtlsdr") + DUMP978_CMD+=("--sdr driver=rtlsdr") fi ;; stratuxv3) if [[ -n "${DUMP978_STRATUXV3_DEVICE}" ]]; then - DUMP978_CMD+=("--stratuxv3" "${DUMP978_STRATUXV3_DEVICE}") + DUMP978_CMD+=("--stratuxv3 ${DUMP978_STRATUXV3_DEVICE}") else "${s6wrap[@]}" echo "ERROR: Please set DUMP978_STRATUXV3_DEVICE!" sleep 60 @@ -47,13 +47,20 @@ if [[ -n "$DUMP978_SDR_AGC" ]]; then fi # Handle "--sdr-gain" -if [[ -e "$GAIN_VALUE_FILE" ]]; then - DUMP978_CMD+=("--sdr-gain" "$(cat "$GAIN_VALUE_FILE")") +READSB_SDR_GAIN="${DUMP978_SDR_GAIN:-${READSB_SDR_GAIN}}" +if [[ -n "$READSB_SDR_GAIN" ]]; then + if [[ "${READSB_SDR_GAIN,,}" == "autogain" ]] && [[ -f /var/globe_history/autogain/gain ]]; then + read -r gain < /var/globe_history/autogain/gain + DUMP978_CMD+=("--sdr-gain $gain") + elif [[ -n "$READSB_SDR_GAIN" ]]; then + [[ "${READSB_SDR_GAIN,,}" == "autogain" ]] && gain="${DUMP978_AUTOGAIN_INITIAL_GAIN:-${READSB_AUTOGAIN_INITIAL_GAIN:-49.6}}" || gain="${READSB_SDR_GAIN}" + DUMP978_CMD+=("--sdr-gain $gain") + fi fi # Handle "--sdr-ppm" if [[ -n "$DUMP978_SDR_PPM" ]]; then - DUMP978_CMD+=("--sdr-ppm" "$DUMP978_SDR_PPM") + DUMP978_CMD+=("--sdr-ppm $DUMP978_SDR_PPM") fi # Handle "--json-stdout" @@ -61,9 +68,15 @@ if [[ -n "$DUMP978_JSON_STDOUT" ]]; then DUMP978_CMD+=("--json-stdout") fi -set -eo pipefail +# do some stuff "just in case" +mkdir -p /run/stats +touch /run/stats/stats.json +touch /run/stats/polar_range.influx -# shellcheck disable=SC2016 -exec "${s6wrap[@]}" "${DUMP978_BIN}" "${DUMP978_CMD[@]}" +# start dump978-fa +set -eo pipefail +"${s6wrap[@]}" echo "invoking: ${DUMP978_BIN} ${DUMP978_CMD[*]}" +# shellcheck disable=SC2068 +exec "${s6wrap[@]}" ${DUMP978_BIN} ${DUMP978_CMD[@]} sleep 5 diff --git a/rootfs/etc/s6-overlay/scripts/readsb b/rootfs/etc/s6-overlay/scripts/readsb index da8cc81..e67144c 100755 --- a/rootfs/etc/s6-overlay/scripts/readsb +++ b/rootfs/etc/s6-overlay/scripts/readsb @@ -3,7 +3,19 @@ set -eo pipefail +s6wrap=(s6wrap --quiet --timestamps --prepend="$(basename "$0")" --args) + sleep 2 -# shellcheck disable=SC2016 -s6wrap --quiet --timestamps --prepend="$(basename "$0")" --args readsb --net-only --net --quiet --net-connector 127.0.0.1,30978,uat_in --net-ro-port 37981 --net-bo-port 37982 +READSB_BIN="readsb" +READSB_ARGS=("--net-only") +READSB_ARGS+=("--net") +READSB_ARGS+=("--quiet") +READSB_ARGS+=("--net-connector 127.0.0.1,30978,uat_in") +READSB_ARGS+=("--net-ro-port 37981") +READSB_ARGS+=("--net-bo-port 37982") + +"${s6wrap[@]}" echo "invoking: ${READSB_BIN} ${READSB_ARGS[*]}" + +# shellcheck disable=SC2068 +exec "${s6wrap[@]}" $READSB_BIN ${READSB_ARGS[@]} diff --git a/rootfs/etc/s6-overlay/scripts/skyaware978 b/rootfs/etc/s6-overlay/scripts/skyaware978 index a35e032..0e65579 100755 --- a/rootfs/etc/s6-overlay/scripts/skyaware978 +++ b/rootfs/etc/s6-overlay/scripts/skyaware978 @@ -3,6 +3,8 @@ set -eo pipefail +s6wrap=(s6wrap --quiet --timestamps --prepend="$(basename "$0")" --args) + sleep 5 # patch skyaware978 to remove the "v=2?" from the URL @@ -11,10 +13,19 @@ sed -i 's/flightFeederCheck();//g' /usr/share/dump978-fa/html/script.js sed -i 's/setStatsLink();//g' /usr/share/dump978-fa/html/script.js sed -i 's/