Offline analysis updates for multimodal fusion. (#360)

Author: celikbasak

.gitignore

@@ -37,4 +37,4 @@ bcipy/language/out/
 
 
 bcipy/simulator/tests/resource/
-!bcipy/simulator/data
+!bcipy/simulator/data

CHANGELOG.md

@@ -34,11 +34,14 @@ Our final release candidate before the official 2.0 release!
     - session data to VEP calibration #322
 - Model
     - Offline analysis to support multimodal fusion. Initial release of GazeModel, GazeReshaper, and Gaze Visualization #294
+    - Updates to ensure seamless offline analysis for both EEG and Gaze data #305
+    - Offline analysis support for EEG and (multiple) gaze models. Updates to support Eye Tracker Evidence class #360
 - Language Model
   - Add Oracle model #316
   - Random Uniform model #311
 - Stimuli
     - Updates to ensure stimuli are presented at the same frequency #287 
+    - Output stimuli position, screen capture and monitor information after Matrix tasks #303
 - Dynamic Selection Window
     - Updated trial_length to trial_window to allow for greater control of window used after stimulus presentations #291
 - Report

bcipy/helpers/demo/demo_visualization.py

@@ -97,9 +97,6 @@
         trigger_timing,
         labels,
         trial_window,
-        transform=default_transform,
-        plot_average=True,
-        plot_topomaps=True,
         save_path=save_path,
         show=args.show
     )

bcipy/helpers/stimuli.py

@@ -24,6 +24,7 @@
 from bcipy.config import DEFAULT_FIXATION_PATH, DEFAULT_TEXT_FIXATION, SESSION_LOG_FILENAME
 from bcipy.exceptions import BciPyCoreException
 from bcipy.helpers.list import grouper
+from bcipy.helpers.symbols import alphabet
 
 # Prevents pillow from filling the console with debug info
 logging.getLogger('PIL').setLevel(logging.WARNING)
@@ -198,7 +199,7 @@ def extract_trials(
             prestimulus_samples: int = 0) -> np.ndarray:
         """Extract Trials.
 
-        After using the InquiryReshaper, it may be necessary to further trial the data for processing.
+        After using the InquiryReshaper, it may be necessary to further extract the trials for processing.
         Using the number of samples and inquiry timing, the data is reshaped from Channels, Inquiry, Samples to
         Channels, Trials, Samples. These should match with the trials extracted from the TrialReshaper given the same
         slicing parameters.
@@ -245,38 +246,46 @@ def __call__(self,
                  target_symbols: List[str],
                  gaze_data: np.ndarray,
                  sample_rate: int,
-                 symbol_set: List[str],
+                 stimulus_duration: float,
+                 num_stimuli_per_inquiry: int,
+                 symbol_set: List[str] = alphabet(),
                  channel_map: Optional[List[int]] = None,
-                 ) -> dict:
-        """Extract inquiry data and labels. Different from the EEG inquiry, the gaze inquiry window starts with
-        the first flicker and ends with the last flicker in the inquiry. Each inquiry has a length of ~3 seconds.
-        The labels are provided in the target_symbols list. It returns a Dict, where keys are the target symbols and
-        the values are inquiries (appended in order of appearance) where the corresponding target symbol is prompted.
+                 ) -> Tuple[dict, list, List[str]]:
+        """Extract gaze trajectory data and labels.
+
+        Different from the EEG, gaze inquiry windows start with the first highlighted symbol and end with the
+        last highlighted symbol in the inquiry. Each inquiry has a length of (trial duration x num of trials)
+        seconds. Labels are provided in 'target_symbols'. It returns a Dict, where keys are the target symbols
+        and the values are inquiries (appended in order of appearance) where the corresponding target symbol is
+        prompted.
+
         Optional outputs:
-        reshape_data is the list of data reshaped into (Inquiries, Channels, Samples), where inquirires are appended
-        in chronological order. labels returns the list of target symbols in each inquiry.
+        reshape_data is the list of data reshaped into (Inquiries, Channels, Samples), where inquirires are
+        appended in chronological order.
+        labels returns the list of target symbols in each inquiry.
 
-        Args:
+        Parameters
+        ----------
             inq_start_times (List[float]): Timestamp of each event in seconds
             target_symbols (List[str]): Prompted symbol in each inquiry
             gaze_data (np.ndarray): shape (channels, samples) eye tracking data
-            sample_rate (int): sample rate of data provided in eeg_data
+            sample_rate (int): sample rate of eye tracker data
+            stimulus_duration (float): duration of flash time (in seconds) for each trial
+            num_stimuli_per_inquiry (int): number of stimuli in each inquiry (default: 10)
+            symbol_set (List[str]): list of all symbols for the task
             channel_map (List[int], optional): Describes which channels to include or discard.
                 Defaults to None; all channels will be used.
 
-        Returns:
-            data_by_targets (dict): Dictionary where keys are the symbol set and values are the appended inquiries
-            for each symbol. dict[Key] = (np.ndarray) of shape (Channels, Samples)
+        Returns
+        -------
+            data_by_targets (dict): Dictionary where keys consist of the symbol set, and values
+            the appended inquiries for each symbol. dict[Key] = (np.ndarray) of shape (Channels, Samples)
 
             reshaped_data (List[float]) [optional]: inquiry data of shape (Inquiries, Channels, Samples)
             labels (List[str]) [optional] : Target symbol in each inquiry.
         """
-        if channel_map:
-            # Remove the channels that we are not interested in
-            channels_to_remove = [idx for idx, value in enumerate(channel_map) if value == 0]
-            gaze_data = np.delete(gaze_data, channels_to_remove, axis=0)
-
-        # Find the value closest to (& greater than) inq_start_times
+        # Find the timestamp value closest to (& greater than) inq_start_times.
+        # Lsl timestamps are the last row in the gaze_data
         gaze_data_timing = gaze_data[-1, :].tolist()
 
         start_times = []
@@ -294,54 +303,45 @@ def __call__(self,
 
         # Create a dictionary with symbols as keys and data as values
         # 'A': [], 'B': [] ...
-        data_by_targets: Dict[str, list] = {}
+        data_by_targets_dict: Dict[str, list] = {}
         for symbol in symbol_set:
-            data_by_targets[symbol] = []
+            data_by_targets_dict[symbol] = []
 
-        window_length = 3  # seconds, total length of flickering after prompt for each inquiry
+        buffer = stimulus_duration / 5  # seconds, buffer for each inquiry
+        # NOTE: This buffer is used to account for the screen downtime between each stimulus.
+        # There is a "duty cycle" of 80% for the stimuli, so we add a buffer of 20% of the stimulus length
+        window_length = (stimulus_duration + buffer) * num_stimuli_per_inquiry   # in seconds
 
         reshaped_data = []
         # Merge the inquiries if they have the same target letter:
         for i, inquiry_index in enumerate(triggers):
             start = inquiry_index
-            stop = int(inquiry_index + (sample_rate * window_length))   # (60 samples * 3 seconds)
+            stop = int(inquiry_index + (sample_rate * window_length))
             # Check if the data exists for the inquiry:
             if stop > len(gaze_data[0, :]):
                 continue
-
-            reshaped_data.append(gaze_data[:, start:stop])
             # (Optional) extracted data (Inquiries x Channels x Samples)
+            reshaped_data.append(gaze_data[:, start:stop])
 
-            # Populate the dict by appending the inquiry to the correct key:
-            data_by_targets[labels[i]].append(gaze_data[:, start:stop])
-
-        # After populating, flatten the arrays in the dictionary to (Channels x Samples):
-        for symbol in symbol_set:
-            if len(data_by_targets[symbol]) > 0:
-                data_by_targets[symbol] = np.transpose(np.array(data_by_targets[symbol]), (1, 0, 2))
-                data_by_targets[symbol] = np.reshape(data_by_targets[symbol], (len(data_by_targets[symbol]), -1))
-
-            # Note that this is a workaround to the issue of having different number of targetness in
-            # each symbol. If a target symbol is prompted more than once, the data is appended to the dict as a list.
-            # Which is why we need to convert it to a (np.ndarray) and flatten the dimensions.
-            # This is not ideal, but it works for now.
+            # Populate the dict by appending the inquiry to the corresponding key:
+            data_by_targets_dict[labels[i]].append(gaze_data[:, start:stop])
 
-        # return np.stack(reshaped_data, 0), labels
-        return data_by_targets
+        return data_by_targets_dict, reshaped_data, labels
 
-    @staticmethod
-    def centralize_all_data(data, symbol_pos):
+    def centralize_all_data(self, data: np.ndarray, symbol_pos: np.ndarray) -> np.ndarray:
         """ Using the symbol locations in matrix, centralize all data (in Tobii units).
         This data will only be used in certain model types.
         Args:
-            data (np.ndarray): Data in shape of num_channels x num_samples
+            data (np.ndarray): Data in shape of num_samples x num_dimensions
             symbol_pos (np.ndarray(float)): Array of the current symbol posiiton in Tobii units
         Returns:
-            data (np.ndarray): Centralized data in shape of num_channels x num_samples
+            new_data (np.ndarray): Centralized data in shape of num_samples x num_dimensions
         """
+        new_data = np.copy(data)
         for i in range(len(data)):
-            data[i] = data[i] - symbol_pos
-        return data
+            new_data[i] = data[i] - symbol_pos
+
+        return new_data
 
 
 class TrialReshaper(Reshaper):