diff --git a/docs/README-airt.md b/docs/README-airt.md index 528f1090..f0a79f5d 100644 --- a/docs/README-airt.md +++ b/docs/README-airt.md @@ -122,3 +122,43 @@ gr::log :DEBUG: retune_fft0 - retuning to {} ``` while in operation - this is normal, the broken logging is because gamutrf/gr-iqtlabs are written for gnuradio 3.10, not for gnuradio 3.9 style logging. + + +## inference with torchserve + +On a non-AIRT machine that the AIRT can reach over the network, that has an nvidia GPU and docker installed: + +# docker nvidia support + +See https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html + +# start torchserve + +From gamutRF's source directory: + +``` +$ mkdir /tmp/torchserve +$ cp torchserve/config.properities /tmp/torchserve +$ docker run --gpus all -p 8081:8081 -p 8080:8080 -v /tmp/torchserve:/torchserve -d iqtlabs/gamutrf-cuda-torchserve torchserve --start --model-store /torchserve --ts-config /torchserve/config.properties --ncs --foreground +``` + +# create and register model + +From gamutRF's source directory, and having obtained mini2_snr.pt: + +``` +$ pip3 install torch-model-archiver +$ torch-model-archiver --force --model-name mini2_snr --version 1.0 --serialized-file /PATH/TO/mini2_snr.pt --handler torchserve/custom_handler.py --export-path /tmp/torchserve +$ curl -X POST "localhost:8081/models?model_name=mini2_snr&url=mini2_snr.mar&initial_workers=4&batch_size=2" +``` + +Now, when starting the scanner, on the AIRT: + +``` +$ LD_PRELOAD=$HOME/.conda/envs/$CONDA_DEFAULT_ENV/lib/libgomp.so.1 gamutrf-scan --sdr=SoapyAIRT --freq-start=300e6 --freq-end=6e9 --tune-step-fft 256 --samp-rate=100e6 --nfft 256 --pretune --no-tag-now --inference_model_server TORCHSERVEHOSTNAME:8080 --inference_model_name mini2_snr --inference_output_dir /tmp --inference_min_db -100 --inference_min_confidence 0.5 +``` + +# inference caveats + +* torchserve can run in CPU-only mode using the same container if no GPU is available, but inference performance will be poor. +* running torchserve and gamutrf on the same host is not currently recommended (due to CPU resource competition, even when a GPU is available). diff --git a/gamutrf/gryolo.py b/gamutrf/gryolo.py index 3fbfcb6a..723be6ee 100644 --- a/gamutrf/gryolo.py +++ b/gamutrf/gryolo.py @@ -56,7 +56,7 @@ def work(self, input_items, output_items): return n def draw_bounding_box(self, img, name, confidence, x, y, x_plus_w, y_plus_h): - label = f"{name}" + label = f"{name}: {confidence}" color = (255, 255, 255) cv2.rectangle(img, (x, y), (x_plus_w, y_plus_h), color, 2) cv2.putText( @@ -72,21 +72,25 @@ def process_item(self, item): scores = [] detections = [] - for name, prediction_data in predictions.items(): - for prediction in prediction_data: - conf = prediction["conf"] - if conf < self.confidence_threshold: - continue - xywh = prediction["xywh"] - box = [ - xywh[0] - (0.5 * xywh[2]), - xywh[1] - (0.5 * xywh[3]), - xywh[2], - xywh[3], - ] - detections.append({"box": box, "score": conf, "name": name}) - boxes.append(box) - scores.append(conf) + try: + for name, prediction_data in predictions.items(): + for prediction in prediction_data: + conf = prediction["conf"] + if conf < self.confidence_threshold: + continue + xywh = prediction["xywh"] + box = [ + xywh[0] - (0.5 * xywh[2]), + xywh[1] - (0.5 * xywh[3]), + xywh[2], + xywh[3], + ] + detections.append({"box": box, "score": conf, "name": name}) + boxes.append(box) + scores.append(conf) + except TypeError as e: + print(f"invalid predictions from torchserve: {e}, {predictions}") + return if not detections: return diff --git a/torchserve/config.properties b/torchserve/config.properties new file mode 100644 index 00000000..c0dd1d46 --- /dev/null +++ b/torchserve/config.properties @@ -0,0 +1,3 @@ +inference_address=http://0.0.0.0:8080 +management_address=http://0.0.0.0:8081 +metrics_address=http://0.0.0.0:8082