Bring master into Timescale branch. (#87)

* Changes for SCMN deployment. * Added Wombat config script info to webapp. * Wombat page changes - fixed log upload script, added data upload command. * Fixed update mapping functions. * ..New tap when open Map * Added axistech poller, changed logging in docker compose. * Adding the axistech module. * Changed axistech message format. * Changed handleSubmit to check if event parameter was supplied. * Logical mapper drops messages whose timestamp is too far into the future. The REST API messages endpoint can now take a physical or logical device id. * Fixed pytest unit tests for the /message REST API endpoint. * AxisTech API only accepts a specific timezone suffix, so changed to use that. * Added the ability to include logical device properties in the ld ls command via the --properties flag. * DB init script now creates version table so DB v1 backups restore cleanly. * Added the ability to include logical device properties in the ld ls command via the --properties flag. DAO & RestAPI updated to handle PostGIS geometry location column. Removed backoff annotations from DAO, they have not been useful. Changed errors raised by the DAO to be ValueErrors when parameter validation fails. * Restoring lost changes to files. * Restoring lost upgrade script. * Webapp changes to handle the geometry location columns. * Removed unused variable. * Fixed webapp map code to handle location lat or long being None, changed docker compose service restart and logging policies. * Issue 82 (#83) Implements issue #82 * More PostGIS and axistech fixes from SCMN (#86) * Issue 82 (#83) (#85) Implements issue #82 * Added extra error logging to axistech receiver. --------- Co-authored-by: dajtxx <[email protected]> * Added Extract.py to read timeseries data from IoTa database. * Fix typo in axistech dependency * Fix the typo again. --------- Co-authored-by: dpi-group <[email protected]> Co-authored-by: Mal Kul <mkul@griffin-water> Co-authored-by: mal-22 <[email protected]> Co-authored-by: mal-22 <[email protected]> Co-authored-by: dajtxx <[email protected]>
DPIclimate · Jul 16, 2024 · 281b9bc · 281b9bc
1 parent 1cb0487
commit 281b9bc
Show file tree

Hide file tree

Showing 27 changed files with 1,364 additions and 482 deletions.
diff --git a/compose/.env b/compose/.env
@@ -47,3 +47,5 @@ TZ=Australia/Sydney
 
 # Used by the Intersect DataBolt delivery service.
 DATABOLT_SHARED_DIR=/some/where/raw_data
+
+AXISTECH_TOKEN=
diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml
@@ -2,8 +2,12 @@ version: '3.1'
 
 services:
   db:
-    image: postgres:14.2
-    restart: "no"
+    image: postgis/postgis:14-3.4
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     volumes:
@@ -18,7 +22,11 @@ services:
   mq:
     hostname: "mq"
     image: rabbitmq:3.9-management
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     volumes:
@@ -30,36 +38,45 @@ services:
 
   restapi:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     depends_on:
       - "db"
     volumes:
       - ../src/python:/home/broker/python
     working_dir: "/home/broker/python/restapi"
-    entrypoint: [ "/home/broker/.local/bin/uvicorn", "--host", "0.0.0.0", "--port", "5687", "RestAPI:app", "--reload" ]
-    healthcheck:
-      test: curl --fail -I http://localhost:5687/openapi.json || exit 1
-      interval: 1m
-      start_period: 20s
+    entrypoint: [ "/home/broker/.local/bin/uvicorn", "--proxy-headers", "--host", "0.0.0.0", "--port", "5687", "RestAPI:app" ]
 
   website:
     image: broker/mgmt-app
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     environment:
       - PYTHONPATH=/app:/iota
     depends_on:
-      restapi:
+      db:
         condition: "service_healthy"
     volumes:
       - ../src/www:/app
       - ../src/python:/iota
 
   ttn_webhook:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -74,7 +91,11 @@ services:
 
   ttn_processor:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -93,7 +114,11 @@ services:
 
   ttn_decoder:
     image: broker/ttn_decoder
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -106,7 +131,11 @@ services:
 
   ydoc:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     profiles:
@@ -123,11 +152,14 @@ services:
 
   wombat:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
-    profiles:
-      - wombat
+    profiles: ["wombat", "frred"]
     depends_on:
       db:
         condition: "service_healthy"
@@ -140,7 +172,11 @@ services:
 
   lm:
     image: broker/python-base
-    restart: "no"
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
     env_file:
       - .env
     depends_on:
@@ -155,6 +191,10 @@ services:
 
   delivery:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -172,6 +212,10 @@ services:
 
   pollers:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -189,6 +233,10 @@ services:
 
   frred:
     image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
     restart: unless-stopped
     env_file:
       - .env
@@ -201,10 +249,26 @@ services:
         condition: "service_healthy"
     volumes:
       - ../src/python:/home/broker/python
-      - ${DATABOLT_SHARED_DIR}/raw_data:/raw_data
+      - ${DATABOLT_SHARED_DIR}/nectar_raw_data:/raw_data
     working_dir: "/home/broker/python"
     entrypoint: [ "python", "-m", "delivery.FRRED" ]
 
+
+  axistech:
+    image: broker/python-base
+    logging:
+      driver: local
+      options:
+        max-file: "3"
+    restart: unless-stopped
+    env_file:
+      - .env
+    profiles:
+      - frred
+    depends_on:
+      - db
+    entrypoint: [ "python", "-m", "pollers.axistech" ]
+
   timescaledb:
     build: ../timescale  # Point to the directory containing the custom Dockerfile
     hostname: "tsdb"
@@ -239,5 +303,3 @@ services:
       - ../src/python:/home/broker/python
     working_dir: "/home/broker/python"
     entrypoint: [ "python", "-m", "timescale.TS_LTSReader" ]
-
-
diff --git a/compose/production/prod.yml b/compose/production/prod.yml
@@ -13,7 +13,7 @@ services:
       # nginx will accept a connection on 1883, forward it to
       # 127.0.0.1:1884, which docker will forward to port 1883
       # within the container.
-      - "127.0.0.1:1884:1883"
+      - "0.0.0.0:1883:1883"
       - "127.0.0.1:15672:15672"
       - "127.0.0.1:15692:15692"
     volumes:

diff --git a/db/init.d/init_db.sql b/db/init.d/init_db.sql
@@ -1,3 +1,6 @@
+CREATE EXTENSION postgis;
+CREATE EXTENSION pgcrypto;
+
 create table if not exists sources (
     source_name text primary key not null
 );
@@ -6,7 +9,7 @@ create table if not exists physical_devices (
     uid integer generated always as identity primary key,
     source_name text not null references sources,
     name text not null,
-    location point,
+    location geometry('POINT', 4283),
     last_seen timestamptz,
     -- Store only top level key value pairs here; it is used
     -- for quickly finding a device using information carried
@@ -72,7 +75,7 @@ create table if not exists device_blobs (
 create table if not exists logical_devices (
     uid integer generated always as identity primary key,
     name text not null,
-    location point,
+    location geometry('POINT', 4283),
     last_seen timestamptz,
     properties jsonb not null default '{}'
 );
@@ -90,7 +93,7 @@ create table if not exists physical_logical_map (
     primary key(physical_uid, logical_uid, start_time)
 );
 
-create table if not exists users(
+create table if not exists users (
     uid integer generated always as identity primary key,
     username text not null unique,
     salt text not null,
@@ -100,6 +103,15 @@ create table if not exists users(
     read_only boolean default True not null
 );
 
+create table if not exists version (
+    version integer not null
+);
+
+create index if not exists pd_src_id_idx on physical_devices using GIN (source_ids);
+
+insert into sources values ('ttn'), ('greenbrain'), ('wombat'), ('ydoc'), ('ict_eagleio');
+insert into version values (2);
+
 create table if not exists data_name_map(
     input_name text not null primary key,
     std_name text not null
@@ -448,8 +460,3 @@ insert into word_list values
             ('UP'),
             ('VAPOUR'),
             ('WIND');
-
--- Enable the PostGIS extensions
--- CREATE EXTENSION postgis;
--- CREATE EXTENSION postgis_raster;
--- CREATE EXTENSION postgis_sfcgal;
diff --git a/db/upgrade/002.sql b/db/upgrade/002.sql
@@ -0,0 +1,17 @@
+CREATE EXTENSION postgis;
+CREATE EXTENSION pgcrypto;
+
+SELECT AddGeometryColumn('logical_devices','geom',4283,'POINT',2);
+SELECT AddGeometryColumn('physical_devices','geom',4283,'POINT',2);
+
+UPDATE logical_devices SET geom = ST_MakePoint(location[1], location[0]) WHERE location IS NOT NULL;
+UPDATE physical_devices SET geom = ST_MakePoint(location[1], location[0]) WHERE location IS NOT NULL;
+
+ALTER TABLE logical_devices DROP COLUMN location;
+ALTER TABLE physical_devices DROP COLUMN location;
+
+ALTER TABLE logical_devices RENAME COLUMN geom TO location;
+ALTER TABLE physical_devices RENAME COLUMN geom TO location;
+
+TRUNCATE version;
+insert into version values (2);
diff --git a/dc.sh b/dc.sh
@@ -12,4 +12,4 @@ if [ "$RUN_MODE" != test ]; then
     fi
 fi
 
-exec docker compose --profile ttn --profile ydoc --profile wombat --profile ubidots --profile pollers --profile frred -p $RUN_MODE -f ../docker-compose.yml -f ./$RUN_MODE.yml $*
+exec docker-compose --profile wombat --profile ubidots --profile frred -p $RUN_MODE -f ../docker-compose.yml -f ./$RUN_MODE.yml $*
diff --git a/images/restapi/requirements.txt b/images/restapi/requirements.txt
@@ -6,29 +6,38 @@ charset-normalizer==2.1.0
 click==8.1.3
 dnspython==2.2.1
 email-validator==1.2.1
+exceptiongroup==1.1.1
 fastapi==0.98.0
 h11==0.13.0
 httpcore==0.17.2
 httptools==0.4.0
 httpx==0.24.1
 idna==3.3
+iniconfig==2.0.0
 itsdangerous==2.1.2
 Jinja2==3.1.2
 MarkupSafe==2.1.1
+numpy==1.26.4
 orjson==3.7.7
+packaging==23.1
+pandas==2.2.1
 pika==1.3.0
+pluggy==1.2.0
 psycopg2-binary==2.9.3
 pydantic==1.9.1
 pytest==7.4.0
 python-dateutil==2.8.2
 python-dotenv==0.20.0
 python-multipart==0.0.5
+pytz==2024.1
 PyYAML==6.0
 requests==2.28.1
 six==1.16.0
 sniffio==1.2.0
 starlette==0.27.0
+tomli==2.0.1
 typing_extensions==4.3.0
+tzdata==2024.1
 ujson==5.4.0
 urllib3==1.26.10
 uvicorn==0.17.6

diff --git a/run.sh b/run.sh
@@ -14,7 +14,7 @@ fi
 
 cd $BROKER_ROOT
 cd compose/$MODE
-./dc.sh stop
+./dc.sh down
 cd $BROKER_ROOT
 docker build -t broker/python-base -f images/restapi/Dockerfile .
 docker build -t broker/ttn_decoder -f images/ttn_decoder/Dockerfile .

diff --git a/src/python/BrokerConstants.py b/src/python/BrokerConstants.py
@@ -13,6 +13,7 @@
 WOMBAT = 'wombat'
 YDOC = 'ydoc'
 ICT_EAGLEIO = 'ict_eagleio'
+AXISTECH = 'axistech'
 
 CREATION_CORRELATION_ID_KEY = 'creation_correlation_id'
 SENSOR_GROUP_ID_KEY = 'sensor_group_id'
Original file line number	Diff line number	Diff line change
Expand Up		@@ -47,3 +47,5 @@ TZ=Australia/Sydney

		# Used by the Intersect DataBolt delivery service.
		DATABOLT_SHARED_DIR=/some/where/raw_data

		AXISTECH_TOKEN=
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,4 +12,4 @@ if [ "$RUN_MODE" != test ]; then @@
         fi
     fi
-    exec docker compose --profile ttn --profile ydoc --profile wombat --profile ubidots --profile pollers --profile frred -p $RUN_MODE -f ../docker-compose.yml -f ./$RUN_MODE.yml $*
+    exec docker-compose --profile wombat --profile ubidots --profile frred -p $RUN_MODE -f ../docker-compose.yml -f ./$RUN_MODE.yml $*