Bring master into Timescale branch.

compose/.env

@@ -47,3 +47,5 @@ TZ=Australia/Sydney
 
 # Used by the Intersect DataBolt delivery service.
 DATABOLT_SHARED_DIR=/some/where/raw_data
+
+AXISTECH_TOKEN=

compose/docker-compose.yml

@@ -2,8 +2,12 @@ version: '3.1'
 
 services:
   db:
-    image: postgres:14.2
-    restart: "no"
+    image: postgis/postgis:14-3.4
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     volumes:
@@ -18,7 +22,11 @@ services:
   mq:
     hostname: "mq"
     image: rabbitmq:3.9-management
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     volumes:
@@ -30,36 +38,45 @@ services:
 
   restapi:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     depends_on:
       - "db"
     volumes:
       - ../src/python:/home/broker/python
     working_dir: "/home/broker/python/restapi"
-    entrypoint: [ "/home/broker/.local/bin/uvicorn", "--host", "0.0.0.0", "--port", "5687", "RestAPI:app", "--reload" ]
-    healthcheck:
-      test: curl --fail -I http://localhost:5687/openapi.json || exit 1
-      interval: 1m
-      start_period: 20s
+    entrypoint: [ "/home/broker/.local/bin/uvicorn", "--proxy-headers", "--host", "0.0.0.0", "--port", "5687", "RestAPI:app" ]
 
   website:
     image: broker/mgmt-app
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     environment:
       - PYTHONPATH=/app:/iota
     depends_on:
-      restapi:
+      db:
         condition: "service_healthy"
     volumes:
       - ../src/www:/app
       - ../src/python:/iota
 
   ttn_webhook:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -74,7 +91,11 @@ services:
 
   ttn_processor:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -93,7 +114,11 @@ services:
 
   ttn_decoder:
     image: broker/ttn_decoder
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -106,7 +131,11 @@ services:
 
   ydoc:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -123,11 +152,14 @@ services:
 
   wombat:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
-    profiles:
-      - wombat
+    profiles: ["wombat", "frred"]
     depends_on:
       db:
         condition: "service_healthy"
@@ -140,7 +172,11 @@ services:
 
   lm:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     depends_on:
@@ -155,6 +191,10 @@ services:
 
   delivery:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -172,6 +212,10 @@ services:
 
   pollers:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -189,6 +233,10 @@ services:
 
   frred:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -201,10 +249,26 @@ services:
         condition: "service_healthy"
     volumes:
       - ../src/python:/home/broker/python
-      - ${DATABOLT_SHARED_DIR}/raw_data:/raw_data
+      - ${DATABOLT_SHARED_DIR}/nectar_raw_data:/raw_data
     working_dir: "/home/broker/python"
     entrypoint: [ "python", "-m", "delivery.FRRED" ]
 
+
+  axistech:
+    image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
+    env_file:
+      - .env
+    profiles:
+      - frred
+    depends_on:
+      - db
+    entrypoint: [ "python", "-m", "pollers.axistech" ]
+
   timescaledb:
     build: ../timescale  # Point to the directory containing the custom Dockerfile
     hostname: "tsdb"
@@ -239,5 +303,3 @@ services:
       - ../src/python:/home/broker/python
     working_dir: "/home/broker/python"
     entrypoint: [ "python", "-m", "timescale.TS_LTSReader" ]
-
-

compose/production/prod.yml

@@ -13,7 +13,7 @@ services:
       # nginx will accept a connection on 1883, forward it to
       # 127.0.0.1:1884, which docker will forward to port 1883
       # within the container.
-      - "127.0.0.1:1884:1883"
+      - "0.0.0.0:1883:1883"
       - "127.0.0.1:15672:15672"
       - "127.0.0.1:15692:15692"
     volumes:

db/init.d/init_db.sql

@@ -1,3 +1,6 @@
+CREATE EXTENSION postgis;
+CREATE EXTENSION pgcrypto;
+
 create table if not exists sources (
     source_name text primary key not null
 );
@@ -6,7 +9,7 @@ create table if not exists physical_devices (
     uid integer generated always as identity primary key,
     source_name text not null references sources,
     name text not null,
-    location point,
+    location geometry('POINT', 4283),
     last_seen timestamptz,
     -- Store only top level key value pairs here; it is used
     -- for quickly finding a device using information carried
@@ -72,7 +75,7 @@ create table if not exists device_blobs (
 create table if not exists logical_devices (
     uid integer generated always as identity primary key,
     name text not null,
-    location point,
+    location geometry('POINT', 4283),
     last_seen timestamptz,
     properties jsonb not null default '{}'
 );
@@ -90,7 +93,7 @@ create table if not exists physical_logical_map (
     primary key(physical_uid, logical_uid, start_time)
 );
 
-create table if not exists users(
+create table if not exists users (
     uid integer generated always as identity primary key,
     username text not null unique,
     salt text not null,
@@ -100,6 +103,15 @@ create table if not exists users(
     read_only boolean default True not null
 );
 
+create table if not exists version (
+    version integer not null
+);
+
+create index if not exists pd_src_id_idx on physical_devices using GIN (source_ids);
+
+insert into sources values ('ttn'), ('greenbrain'), ('wombat'), ('ydoc'), ('ict_eagleio');
+insert into version values (2);
+
 create table if not exists data_name_map(
     input_name text not null primary key,
     std_name text not null
@@ -448,8 +460,3 @@ insert into word_list values
             ('UP'),
             ('VAPOUR'),
             ('WIND');
-
--- Enable the PostGIS extensions
--- CREATE EXTENSION postgis;
--- CREATE EXTENSION postgis_raster;
--- CREATE EXTENSION postgis_sfcgal;

db/upgrade/002.sql

@@ -0,0 +1,17 @@
+CREATE EXTENSION postgis;
+CREATE EXTENSION pgcrypto;
+
+SELECT AddGeometryColumn('logical_devices','geom',4283,'POINT',2);
+SELECT AddGeometryColumn('physical_devices','geom',4283,'POINT',2);
+
+UPDATE logical_devices SET geom = ST_MakePoint(location[1], location[0]) WHERE location IS NOT NULL;
+UPDATE physical_devices SET geom = ST_MakePoint(location[1], location[0]) WHERE location IS NOT NULL;
+
+ALTER TABLE logical_devices DROP COLUMN location;
+ALTER TABLE physical_devices DROP COLUMN location;
+
+ALTER TABLE logical_devices RENAME COLUMN geom TO location;
+ALTER TABLE physical_devices RENAME COLUMN geom TO location;
+
+TRUNCATE version;
+insert into version values (2);

dc.sh

@@ -12,4 +12,4 @@ if [ "$RUN_MODE" != test ]; then
     fi
 fi
 
-exec docker compose --profile ttn --profile ydoc --profile wombat --profile ubidots --profile pollers --profile frred -p $RUN_MODE -f ../docker-compose.yml -f ./$RUN_MODE.yml $*
+exec docker-compose --profile wombat --profile ubidots --profile frred -p $RUN_MODE -f ../docker-compose.yml -f ./$RUN_MODE.yml $*

images/restapi/requirements.txt

@@ -6,29 +6,38 @@ charset-normalizer==2.1.0
 click==8.1.3
 dnspython==2.2.1
 email-validator==1.2.1
+exceptiongroup==1.1.1
 fastapi==0.98.0
 h11==0.13.0
 httpcore==0.17.2
 httptools==0.4.0
 httpx==0.24.1
 idna==3.3
+iniconfig==2.0.0
 itsdangerous==2.1.2
 Jinja2==3.1.2
 MarkupSafe==2.1.1
+numpy==1.26.4
 orjson==3.7.7
+packaging==23.1
+pandas==2.2.1
 pika==1.3.0
+pluggy==1.2.0
 psycopg2-binary==2.9.3
 pydantic==1.9.1
 pytest==7.4.0
 python-dateutil==2.8.2
 python-dotenv==0.20.0
 python-multipart==0.0.5
+pytz==2024.1
 PyYAML==6.0
 requests==2.28.1
 six==1.16.0
 sniffio==1.2.0
 starlette==0.27.0
+tomli==2.0.1
 typing_extensions==4.3.0
+tzdata==2024.1
 ujson==5.4.0
 urllib3==1.26.10
 uvicorn==0.17.6

run.sh

@@ -14,7 +14,7 @@ fi
 
 cd $BROKER_ROOT
 cd compose/$MODE
-./dc.sh stop
+./dc.sh down
 cd $BROKER_ROOT
 docker build -t broker/python-base -f images/restapi/Dockerfile .
 docker build -t broker/ttn_decoder -f images/ttn_decoder/Dockerfile .

src/python/BrokerConstants.py

@@ -13,6 +13,7 @@
 WOMBAT = 'wombat'
 YDOC = 'ydoc'
 ICT_EAGLEIO = 'ict_eagleio'
+AXISTECH = 'axistech'
 
 CREATION_CORRELATION_ID_KEY = 'creation_correlation_id'
 SENSOR_GROUP_ID_KEY = 'sensor_group_id'