multimodal fix, full release!!!

- added locks to psuedo generator.
- fixed multimodal after new thread creation
- added pupil labs channel descriptions to examples

pybci/Examples/PsuedoLSLStreamGenerator/mainSend.py

@@ -44,6 +44,7 @@ class Window(QtWidgets.QWidget):
     def __init__(self, *args, **kwargs):
         super(Window, self).__init__(*args, **kwargs)
         #self.showMaximized()
+        self.lock = threading.Lock()  # Lock for thread safety
         # setting up different signal varables
         self.commandDataConfigs[0].amplitude = 1
         self.commandDataConfigs[0].noise_level = 1
@@ -97,29 +98,34 @@ def __init__(self, *args, **kwargs):
         self.ChangeString()
 
     def update(self):
-        if self.markerOccurred:
-            for i, command in enumerate(self.commandStrings):
-                if self.currentMarker == command:
-                    num_samples = int(self.commandDataConfigs[i].duration/10 * self.sampleRate)
-                    [self.y[0].popleft() for d in range(num_samples)] 
-                    num = self.GeneratePseudoEMG(self.sampleRate,self.commandDataConfigs[i].duration/10, self.commandDataConfigs[i].noise_level, 
-                                                    self.commandDataConfigs[i].amplitude, self.commandDataConfigs[i].frequency)
-            self.chunkCount += 1
-            if self.chunkCount >= 10:
-                self.markerOccurred = False
-                self.chunkCount = 0 
-        else:# send baseline
-            num_samples = int(self.baselineConfig.duration/10 * self.sampleRate)
-            [self.y[0].popleft() for d in range(num_samples)] 
-            num = self.GeneratePseudoEMG(self.sampleRate,self.baselineConfig.duration/10, self.baselineConfig.noise_level, 
-                                            self.baselineConfig.amplitude, self.baselineConfig.frequency)
-        [self.y[0].extend([num[0][d]]) for d in range(num_samples)] 
-        for n in range(num.shape[1]):
-            self.outlet.push_sample(num[:,n])
-
-        #self.outlet.push_chunk(num.)
-
-        self.plot.setData(list(self.x),list(self.y[0]), pen=pg.mkPen(1), clear=True)#pen = self.colours[x], 
+        with self.lock:  # Acquire the lock
+            if self.markerOccurred:
+                for i, command in enumerate(self.commandStrings):
+                    if self.currentMarker == command:
+                        num_samples = int(self.commandDataConfigs[i].duration/10 * self.sampleRate)
+                        [self.y[0].popleft() for d in range(num_samples)] 
+                        num = self.GeneratePseudoEMG(self.sampleRate,self.commandDataConfigs[i].duration/10, self.commandDataConfigs[i].noise_level, 
+                                                        self.commandDataConfigs[i].amplitude, self.commandDataConfigs[i].frequency)
+                self.chunkCount += 1
+                if self.chunkCount >= 10:
+                    self.markerOccurred = False
+                    self.chunkCount = 0 
+            else:# send baseline
+                num_samples = int(self.baselineConfig.duration/10 * self.sampleRate)
+                [self.y[0].popleft() for d in range(num_samples)] 
+                num = self.GeneratePseudoEMG(self.sampleRate,self.baselineConfig.duration/10, self.baselineConfig.noise_level, 
+                                                self.baselineConfig.amplitude, self.baselineConfig.frequency)
+            [self.y[0].extend([num[0][d]]) for d in range(num_samples)] 
+            for n in range(num.shape[1]):
+                self.outlet.push_sample(num[:,n])
+            #print(num.T.shape)
+            #chunk = num.T
+            #print(chunk)
+            #for sample in chunk:
+            #    if len(sample) != 8:
+            #        print(f"Invalid sample: {sample}")
+            #self.outlet.push_chunk(chunk)
+            self.plot.setData(list(self.x),list(self.y[0]), pen=pg.mkPen(1), clear=True)#pen = self.colours[x], 
 
     def GeneratePseudoEMG(self,samplingRate, duration, noise_level, amplitude, frequency):
         """
@@ -171,9 +177,10 @@ def StopTest(self):
         self.button.setText("Start Test")
         self.button.clicked.connect(self.BeginTest)
 
-    def SendMarker(self):   
-        self.markerOutlet.push_sample([self.currentMarker])
-        self.markerOccurred = True
+    def SendMarker(self):
+        with self.lock:  # Acquire the lock
+            self.markerOutlet.push_sample([self.currentMarker])
+            self.markerOccurred = True
 
     def SendBaseline(self):
         self.markerOutlet.push_sample(["Baseline"])

pybci/Examples/PupilLabsRightLeftEyeClose/RightLeftMarkers.py

@@ -24,9 +24,9 @@ def __init__(self, root):
         self.label.grid(row=0, column=0, columnspan=2, padx=20, pady=20, sticky="nsew")
 
         self.button = tk.Button(root, text="Start", command=self.toggle_iteration, font=("Helvetica", 18))
-        self.button.grid(row=1, column=0, padx=20, pady=20, sticky="nsew")
-        self.custom_button = tk.Button(root, text="Start Testing", command=self.custom_function, font=("Helvetica", 18))
-        self.custom_button.grid(row=1, column=1, padx=20, pady=20, sticky="nsew")
+        self.button.grid(row=1, column=0,columnspan=2, padx=20, pady=20, sticky="nsew")
+        #self.custom_button = tk.Button(root, text="Start Testing", command=self.custom_function, font=("Helvetica", 18))
+        #self.custom_button.grid(row=1, column=1, padx=20, pady=20, sticky="nsew")
         self.close_button = tk.Button(root, text="Close", command=self.root.destroy, font=("Helvetica", 18))
         self.close_button.grid(row=2, column=0, columnspan=2, padx=20, pady=20, sticky="nsew")
 
@@ -72,9 +72,9 @@ def remove_stimulus(self, index):
                 self.root.after_cancel(self.after_id)
                 self.after_id = None
 
-    def custom_function(self):
-        # Define your custom function here
-        print("Custom function called")
+    #def custom_function(self):
+    #    # Define your custom function here
+    #    print("Custom function called")
 
 root = tk.Tk()
 app = App(root)

pybci/Examples/PupilLabsRightLeftEyeClose/bciGazeExample.py

@@ -1,7 +1,12 @@
 import time
 from pybci import PyBCI
 import numpy as np
-
+# pupil-labs channels:
+#['confidence',  1
+# 'norm_pos_x', 'norm_pos_y', 'gaze_point_3d_x', 'gaze_point_3d_y', 'gaze_point_3d_z',  + 5
+# 'eye_center0_3d_x', 'eye_center0_3d_y', 'eye_center0_3d_z', 'eye_center1_3d_x', 'eye_center1_3d_y',  + 5
+# 'eye_center1_3d_z', # 'gaze_normal0_x', 'gaze_normal0_y', 'gaze_normal0_z', 'gaze_normal1_x',  + 5
+# 'gaze_normal1_y', 'gaze_normal1_z', 'diameter0_2d', 'diameter1_2d', 'diameter0_3d',  'diameter1_3d'] + 6 = 22 channels
 class PupilGazeDecode():
     def __init__(self):
         super().__init__()
@@ -37,7 +42,7 @@ def ProcessFeatures(self, epochData, sr, epochNum): # This is the required funct
             if min([currentMarkers[key][1] for key in currentMarkers]) > bci.minimumEpochsRequired:
                 classInfo = bci.CurrentClassifierInfo() # hangs if called too early
                 accuracy = classInfo["accuracy"]
-            if min([currentMarkers[key][1] for key in currentMarkers]) > bci.minimumEpochsRequired+12:  
+            if min([currentMarkers[key][1] for key in currentMarkers]) > bci.minimumEpochsRequired+15:  
                 bci.TestMode()
                 break
     while True:

pybci/Examples/testMultimodal.py

@@ -1,27 +1,15 @@
-"""
-WARNING! NEEDS TESTING AFTER NEW DATA RECEIVER THREAD CREATION AND SCALAR APPLICATION
-"""
 import time
 from pybci import PyBCI
 import numpy as np
-import tensorflow as tf# bring in tf for custom model creation
 from pybci.Utils.FeatureExtractor import GenericFeatureExtractor
-# Define our model, must be 1 dimesional at output to flatten due to multi-modal devices
-pupilFeatures = 3
-eegChs = 2 # Fp1, Fp2
-eegfeaturesPerCh = 17
-totalFeatures = (eegfeaturesPerCh * eegChs) + pupilFeatures
-model = tf.keras.models.Sequential([
-    tf.keras.layers.Dense(64, activation='relu', input_shape=(totalFeatures,)),
-    tf.keras.layers.Dense(32, activation='relu'),
-    tf.keras.layers.Dense(1, activation='sigmoid') # 2 class output
-])
-model.compile(
-    optimizer='adam',
-    loss='binary_crossentropy',  # 2 class output
-    metrics=['accuracy']
-)
+
 class PupilGazeDecode():
+    # pupil-labs channels:
+    #['confidence',  1
+    # 'norm_pos_x', 'norm_pos_y', 'gaze_point_3d_x', 'gaze_point_3d_y', 'gaze_point_3d_z',  + 5
+    # 'eye_center0_3d_x', 'eye_center0_3d_y', 'eye_center0_3d_z', 'eye_center1_3d_x', 'eye_center1_3d_y',  + 5
+    # 'eye_center1_3d_z', # 'gaze_normal0_x', 'gaze_normal0_y', 'gaze_normal0_z', 'gaze_normal1_x',  + 5
+    # 'gaze_normal1_y', 'gaze_normal1_z', 'diameter0_2d', 'diameter1_2d', 'diameter0_3d',  'diameter1_3d'] + 6 = 22 channels
     def __init__(self):
         super().__init__()
     def ProcessFeatures(self, epochData, sr, epochNum): # This is the required function name and variables that are passed to all 
@@ -30,19 +18,50 @@ def ProcessFeatures(self, epochData, sr, epochNum): # This is the required funct
         if len(epochData[0]) ==  0:
             return [0,0,0]
         else:
-            rightmean = np.mean(epochData[0]) # channel 20 is 3d pupil diameter right, get mean
-            leftmean = np.mean(epochData[1]) # channel 21 is 3d pupil diameter right, get mean
-            bothmean = np.mean([(epochData[0][i] + epochData[1][i]) / 2 for i in range(len(epochData[0]))]) # mean of both eyes in 3d
-            return np.nan_to_num([rightmean,leftmean,bothmean, len(epochData[0])]) #  expects 2d
+            confidence = np.mean(epochData[0]) # channel 21 is 3d pupil diameter right, get mean
+            rightmean = np.mean(epochData[1]) # channel 20 is 3d pupil diameter right, get mean
+            leftmean = np.mean(epochData[2]) # channel 21 is 3d pupil diameter right, get mean
+            bothmean = np.mean([(epochData[1][i] + epochData[2][i]) / 2 for i in range(len(epochData[1]))]) # mean of both eyes in 3d
+            return np.nan_to_num([confidence, rightmean,leftmean,bothmean]) #  expects 2d
+
+from scipy.fft import fft
+class EOGClassifier():
+    # used Fp1 and Fp2 from io:bio EEG device
+    def ProcessFeatures(self, epochData, sr, epochNum): # Every custom class requires a function with this name and structure to extract the featur data and epochData is always [Samples, Channels]
+        #print(epochData.shape)
+        rmsCh1 = np.sqrt(np.mean(np.array(epochData[:,0])**2))
+        rangeCh1 = max(epochData[:,0])-min(epochData[:,0])
+        varCh1 = np.var(epochData[:,0])
+        meanAbsCh1 = np.mean(np.abs(epochData[:,0]))
+        zeroCrossCh1 = ((epochData[:,0][:-1] * epochData[:,0][1:]) < 0).sum()
+        fft_result = fft(epochData[:,0])
+        frequencies = np.fft.fftfreq(len(epochData[:,0]), 1/ sr)
+        delta_mask = (frequencies >= 0.5) & (frequencies <= 2)
+        delta_power = np.mean(np.abs(fft_result[delta_mask])**2)
+        delta2_mask = (frequencies >= 2) & (frequencies <= 4)
+        delta2_power = np.mean(np.abs(fft_result[delta2_mask])**2)
+        theta_mask = (frequencies >= 4) & (frequencies <= 7)
+        theta_power = np.mean(np.abs(fft_result[theta_mask])**2)
+        alpha_mask = (frequencies >= 7) & (frequencies <= 10)
+        alpha_power = np.mean(np.abs(fft_result[alpha_mask])**2)
+        beta_mask = (frequencies >= 10) & (frequencies <= 15)
+        beta_power = np.mean(np.abs(fft_result[beta_mask])**2)
+        beta2_mask = (frequencies >= 15) & (frequencies <= 20)
+        beta2_power = np.mean(np.abs(fft_result[beta2_mask])**2)
+        gamma_mask = (frequencies >= 20) & (frequencies <= 25)
+        gamma_power = np.mean(np.abs(fft_result[gamma_mask])**2)
+        a = np.array([rmsCh1, varCh1,rangeCh1,  meanAbsCh1,  zeroCrossCh1, max(epochData[:,0]), min(epochData[:,0]), 
+                         alpha_power, delta_power,delta2_power, theta_power, beta_power,beta2_power, gamma_power]).T
+        return np.nan_to_num(a)
 
-hullUniEEGLSLStreamName = "sendTest"#EEGStream"
+hullUniEEGLSLStreamName = "EEGStream"#EEGStream"
 pupilLabsLSLName = "pupil_capture" 
 markerstream = "TestMarkers" # using pupillabs rightleftmarkers example
-streamCustomFeatureExtract = {pupilLabsLSLName: PupilGazeDecode(), hullUniEEGLSLStreamName: GenericFeatureExtractor()}
+streamCustomFeatureExtract = {pupilLabsLSLName: PupilGazeDecode(), hullUniEEGLSLStreamName: EOGClassifier()} #GenericFeatureExtractor
 dataStreamNames = [pupilLabsLSLName, hullUniEEGLSLStreamName]
 # to reduce overall computational complexity we are going to drop irrelevant channels
-streamChsDropDict = {hullUniEEGLSLStreamName : [6,7],#0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,22,23], # for our device we have Fp1 and Fp2 on channels 18 and 19, so list values 17 and 18 removed
-                     pupilLabsLSLName: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] # pupil labs we only wan left and right 3d pupil diameter, drop rest
+streamChsDropDict = {hullUniEEGLSLStreamName : [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,22,23],#0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,19,20,21,22,23], # for our device we have Fp1 and Fp2 on channels 18 and 19, so list values 17 and 18 removed
+                     pupilLabsLSLName: [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17, 20, 21] # pupil labs we only wan left and right 3d pupil diameter, drop rest
                      } 
 bci = PyBCI(dataStreams = dataStreamNames, markerStream=markerstream, minimumEpochsRequired = 4,
             streamChsDropDict = streamChsDropDict,
@@ -63,7 +82,7 @@ def ProcessFeatures(self, epochData, sr, epochNum): # This is the required funct
             if min([currentMarkers[key][1] for key in currentMarkers]) > bci.minimumEpochsRequired:
                 classInfo = bci.CurrentClassifierInfo() # hangs if called too early
                 accuracy = classInfo["accuracy"]
-            if min([currentMarkers[key][1] for key in currentMarkers]) > bci.minimumEpochsRequired+1:  
+            if min([currentMarkers[key][1] for key in currentMarkers]) > bci.minimumEpochsRequired+6:  
                 bci.TestMode()
                 break
     while True:

pybci/ThreadClasses/ClassifierThread.py

@@ -8,7 +8,7 @@ class ClassifierThread(threading.Thread):
     targets = np.array([])
     mode = "train"
     guess = " "
-    epochCountsc = {} 
+    #epochCountsc = {} 
     def __init__(self, closeEvent,trainTestEvent, featureQueueTest,featureQueueTrain, classifierInfoQueue, classifierInfoRetrieveEvent, 
                  classifierGuessMarkerQueue, classifierGuessMarkerEvent, queryFeaturesQueue, queryFeaturesEvent,
                  logger = Logger(Logger.INFO), numStreamDevices = 1,
@@ -30,14 +30,15 @@ def __init__(self, closeEvent,trainTestEvent, featureQueueTest,featureQueueTrain
         self.logger = logger
 
     def run(self):
+        epochCountsc={}
         if self.numStreamDevices > 1:
             tempdatatrain = {}
             tempdatatest = {}
         while not self.closeEvent.is_set():
             if self.trainTestEvent.is_set(): # We're training!
                 if self.featureQueueTrain.empty():
-                    if len(self.epochCountsc) > 1: # check if there is more then one test condition
-                        minNumKeyEpochs = min([self.epochCountsc[key][1] for key in self.epochCountsc]) # check minimum viable number of training eochs have been obtained
+                    if len(epochCountsc) > 1: # check if there is more then one test condition
+                        minNumKeyEpochs = min([epochCountsc[key][1] for key in epochCountsc]) # check minimum viable number of training eochs have been obtained
                         if minNumKeyEpochs < self.minRequiredEpochs:
                             pass
                         else: 
@@ -50,15 +51,17 @@ def run(self):
                         self.classifierGuessMarkerQueue.put(self.guess)
                 else:
                     try:
-                        featuresSingle, devCount, target, self.epochCountsc = self.featureQueueTrain.get_nowait() #[dataFIFOs, self.currentMarker, self.sr, self.dataType]
-
+                        featuresSingle, devCount, target, epochCountsc = self.featureQueueTrain.get_nowait() #[dataFIFOs, self.currentMarker, self.sr, self.dataType]
                         if self.numStreamDevices > 1: # Collects multiple data strems feature sets and synchronise here
                             tempdatatrain[devCount] = featuresSingle
                             if len(tempdatatrain) == self.numStreamDevices:
-                                flattened_list = [item for sublist in tempdatatrain.values() for item in sublist]
+                                flattened_list = np.array([item for sublist in tempdatatrain.values() for item in sublist])
                                 tempdatatrain = {}
                                 self.targets = np.append(self.targets, [target], axis = 0)
-                                self.features = np.append(self.features, [flattened_list], axis = 0)
+                                #self.features = np.append(self.features, [flattened_list], axis = 0)
+                                if self.features.shape[0] == 0:
+                                    self.features = self.features.reshape((0,) + flattened_list.shape)
+                                self.features = np.append(self.features, [flattened_list], axis=0)
                             # need to check if all device data is captured, then flatten and append  
                         else: # Only one device to collect from
                             if self.features.shape[0] == 0:
@@ -74,8 +77,7 @@ def run(self):
                         tempdatatest[devCount] = featuresSingle
                         if len(tempdatatest) == self.numStreamDevices:
                             flattened_list = []
-                            for value in tempdatatest.values():
-                                flattened_list.extend(value)
+                            flattened_list = np.array([item for sublist in tempdatatest.values() for item in sublist])
                             tempdatatest = {}
                             start = time.time()
                             self.guess = self.classifier.TestModel(flattened_list)
@@ -84,7 +86,6 @@ def run(self):
                                 self.logger.log(Logger.TIMING, f" classifier testing time {end - start}")
                     else:
                         start = time.time()
-                        #print(featuresSingle)
                         self.guess = self.classifier.TestModel(featuresSingle)
                         if (self.logger.level == Logger.TIMING):
                             end = time.time()

pybci/ThreadClasses/FeatureProcessorThread.py

@@ -2,6 +2,8 @@
 from ..Utils.FeatureExtractor import GenericFeatureExtractor
 from ..Utils.Logger import Logger
 from ..Configuration.EpochSettings import GlobalEpochSettings
+import copy
+
 class FeatureProcessorThread(threading.Thread):
     tempDeviceEpochLogger = []
     def __init__(self, closeEvent, trainTestEvent, dataQueueTrain,dataQueueTest,
@@ -44,7 +46,7 @@ def run(self):
                         self.logger.log(Logger.TIMING, f" Feature Extraction time {end - start}")
                         if (end-start) >self.globalWindowSettings.windowLength:
                             self.logger.log(Logger.WARNING, f" Feature Extraction time > globalEpochSetting.windowLength, will create lag in classification output. Recommended to reduce channels, smapling rate, and features or reduce feature computational complexity.")
-                    self.featureQueueTrain.put( [features, devCount, target, self.epochCounts] )
+                    self.featureQueueTrain.put( [features, devCount, target, dict(self.epochCounts)] )
                 except queue.Empty:
                     pass
             else: