Merged PR 228: Fix bug that was preventing compression libraries to b…

…e used on time data.
LaussenLabs · Mar 18, 2024 · 7483726 · 7483726
1 parent 395e7f5
commit 7483726
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 74 deletions.
diff --git a/sdk/atriumdb/block.py b/sdk/atriumdb/block.py
@@ -334,6 +334,7 @@ def decode_blocks(self, encoded_bytes, num_bytes_list, analog=True, time_type=1)
         # Calculate the start of the time bytes within the encoded bytes
         t_byte_start = np.cumsum([h.t_encoded_size if h.t_compression != 1 else 0 for h in headers], dtype=np.uint64)
         t_byte_start = np.concatenate([np.array([0], dtype=np.uint64), t_byte_start[:-1]], axis=None)
+        t_byte_start += encoded_bytes.size
         end_bench = time.perf_counter()
         logging.debug(f"arrange intra-block information {(end_bench - start_bench) * 1000} ms")
 

diff --git a/sdk/tests/example_data/mitbih_seed_42_all_devices.yaml b/sdk/tests/example_data/mitbih_seed_42_all_devices.yaml
@@ -1,3 +1,15 @@
+measures:
+- tag: MLII
+  freq_nhz: 500000000000
+  units: mV
+- tag: V5
+  freq_nhz: 500000000000
+  units: mV
+- tag: V1
+  freq_nhz: 500000000000
+  units: mV
+patient_ids: {}
+mrns: {}
 device_ids:
   1: all
   2: all
@@ -14,25 +26,9 @@ labels:
 - Premature Ventricular Contraction
 - Normal Sinus Rhythm
 - Ventricular Fibrillation
-- Premature Atrial Contraction
-- Atrial Flutter
 - Supraventricular Tachycardia
 - Third Degree Heart Block
 - Second Degree Heart Block
 - Junctional Rhythm
+- Atrial Flutter
 - Bundle Branch Block
-- Pause
-- First Degree Heart Block
-- Atrial Fibrillation
-measures:
-- freq_nhz: 500000000000
-  tag: MLII
-  units: mV
-- freq_nhz: 500000000000
-  tag: V5
-  units: mV
-- freq_nhz: 500000000000
-  tag: V1
-  units: mV
-mrns: {}
-patient_ids: {}
diff --git a/sdk/tests/example_data/mitbih_seed_42_all_mrns.yaml b/sdk/tests/example_data/mitbih_seed_42_all_mrns.yaml
@@ -1,3 +1,17 @@
+measures:
+- tag: MLII
+  freq_nhz: 500000000000
+  units: mV
+- tag: V5
+  freq_nhz: 500000000000
+  units: mV
+- tag: V1
+  freq_nhz: 500000000000
+  units: mV
+patient_ids: {}
+mrns:
+  770487: all
+  988662: all
 device_ids: {}
 device_tags: {}
 labels:
@@ -12,27 +26,9 @@ labels:
 - Premature Ventricular Contraction
 - Normal Sinus Rhythm
 - Ventricular Fibrillation
-- Premature Atrial Contraction
-- Atrial Flutter
 - Supraventricular Tachycardia
 - Third Degree Heart Block
 - Second Degree Heart Block
 - Junctional Rhythm
+- Atrial Flutter
 - Bundle Branch Block
-- Pause
-- First Degree Heart Block
-- Atrial Fibrillation
-measures:
-- freq_nhz: 500000000000
-  tag: MLII
-  units: mV
-- freq_nhz: 500000000000
-  tag: V5
-  units: mV
-- freq_nhz: 500000000000
-  tag: V1
-  units: mV
-mrns:
-  498382: all
-  770487: all
-patient_ids: {}
diff --git a/sdk/tests/example_data/mitbih_seed_42_all_patients.yaml b/sdk/tests/example_data/mitbih_seed_42_all_patients.yaml
@@ -1,3 +1,17 @@
+measures:
+- tag: MLII
+  freq_nhz: 500000000000
+  units: mV
+- tag: V5
+  freq_nhz: 500000000000
+  units: mV
+- tag: V1
+  freq_nhz: 500000000000
+  units: mV
+patient_ids:
+  1: all
+  2: all
+mrns: {}
 device_ids: {}
 device_tags: {}
 labels:
@@ -12,27 +26,9 @@ labels:
 - Premature Ventricular Contraction
 - Normal Sinus Rhythm
 - Ventricular Fibrillation
-- Premature Atrial Contraction
-- Atrial Flutter
 - Supraventricular Tachycardia
 - Third Degree Heart Block
 - Second Degree Heart Block
 - Junctional Rhythm
+- Atrial Flutter
 - Bundle Branch Block
-- Pause
-- First Degree Heart Block
-- Atrial Fibrillation
-measures:
-- freq_nhz: 500000000000
-  tag: MLII
-  units: mV
-- freq_nhz: 500000000000
-  tag: V5
-  units: mV
-- freq_nhz: 500000000000
-  tag: V1
-  units: mV
-mrns: {}
-patient_ids:
-  1: all
-  2: all
diff --git a/sdk/tests/example_data/mitbih_seed_42_all_tags.yaml b/sdk/tests/example_data/mitbih_seed_42_all_tags.yaml
@@ -1,3 +1,15 @@
+measures:
+- tag: MLII
+  freq_nhz: 500000000000
+  units: mV
+- tag: V5
+  freq_nhz: 500000000000
+  units: mV
+- tag: V1
+  freq_nhz: 500000000000
+  units: mV
+patient_ids: {}
+mrns: {}
 device_ids: {}
 device_tags:
   '100': all
@@ -14,25 +26,9 @@ labels:
 - Premature Ventricular Contraction
 - Normal Sinus Rhythm
 - Ventricular Fibrillation
-- Premature Atrial Contraction
-- Atrial Flutter
 - Supraventricular Tachycardia
 - Third Degree Heart Block
 - Second Degree Heart Block
 - Junctional Rhythm
+- Atrial Flutter
 - Bundle Branch Block
-- Pause
-- First Degree Heart Block
-- Atrial Fibrillation
-measures:
-- freq_nhz: 500000000000
-  tag: MLII
-  units: mV
-- freq_nhz: 500000000000
-  tag: V5
-  units: mV
-- freq_nhz: 500000000000
-  tag: V1
-  units: mV
-mrns: {}
-patient_ids: {}
diff --git a/sdk/tests/test_mit_bih.py b/sdk/tests/test_mit_bih.py
@@ -26,7 +26,7 @@
 
 DB_NAME = 'atrium-mit-bih'
 
-MAX_RECORDS = 1
+MAX_RECORDS = 4
 SEED = 42
 LABEL_SET_LIST = [
     "Normal Sinus Rhythm",
@@ -221,12 +221,23 @@ def write_to_sdk(freq_nano, device_id, gap_data_2d, time_arr, start_time, sdk, p
     if random.random() < 0.5:
         # Time type 1
         raw_t_t = T_TYPE_TIMESTAMP_ARRAY_INT64_NANO
-        encoded_t_t = T_TYPE_GAP_ARRAY_INT64_INDEX_DURATION_NANO
+
+        if random.random() < 0.5:
+            # Time Value pair encoding
+            encoded_t_t = T_TYPE_TIMESTAMP_ARRAY_INT64_NANO
+            sdk.block.t_compression = 3
+            sdk.block.t_compression_level = 12
+        else:
+            # Gap Array encoding
+            encoded_t_t = T_TYPE_GAP_ARRAY_INT64_INDEX_DURATION_NANO
 
         # Call the write_data method with the determined parameters
         sdk.write_data(measure_id, device_id, time_arr, value_data, freq_nano, start_time,
                        raw_time_type=raw_t_t, raw_value_type=raw_v_t, encoded_time_type=encoded_t_t,
                        encoded_value_type=encoded_v_t, scale_m=scale_m, scale_b=scale_b, gap_tolerance=gap_tolerance)
+
+        sdk.block.t_compression = 1
+        sdk.block.t_compression_level = 0
     else:
         # Time type 2
         raw_t_t = T_TYPE_GAP_ARRAY_INT64_INDEX_DURATION_NANO