Multimodal

CHANGELOG.md

@@ -7,6 +7,8 @@ Our last release candidate before the official 2.0 release!
 - Multimodal Acquisition and Querying
     - Support for multiple devices in online querying #286
     - Support for trigger handling relative to a given device #293
+- Model
+    - Offline analysis of multimodal fusion. First version of gaze model is completed #294 Gaze reshaper is updated #294 Visualization for the gaze classification is added #294
 - Stimuli
     - Updates to ensure stimuli are presented at the same frequency #287
 - Dynamic Selection Window

MANIFEST.in

@@ -0,0 +1,7 @@
+include bcipy/parameters/*.json
+include bcipy/parameters/experiment/*.json
+include bcipy/parameters/field/*.json
+include bcipy/language/lms/*
+include bcipy/language/sets/*
+include bcipy/static/images/*/*
+include bcipy/static/sounds/*/*

bcipy/config.py

@@ -14,8 +14,8 @@
 EXPERIMENT_DATA_FILENAME = 'experiment_data.json'
 BCIPY_ROOT = Path(__file__).resolve().parent
 ROOT = BCIPY_ROOT.parent
-DEFAULT_EXPERIMENT_PATH = f'{ROOT}/.bcipy/experiment'
-DEFAULT_FIELD_PATH = f'{ROOT}/.bcipy/field'
+DEFAULT_EXPERIMENT_PATH = f'{BCIPY_ROOT}/parameters/experiment'
+DEFAULT_FIELD_PATH = f'{BCIPY_ROOT}/parameters/field'
 
 DEFAULT_PARAMETER_FILENAME = 'parameters.json'
 DEFAULT_PARAMETERS_PATH = f'{BCIPY_ROOT}/parameters/{DEFAULT_PARAMETER_FILENAME}'
@@ -28,12 +28,13 @@
 STATIC_IMAGES_PATH = f'{STATIC_PATH}/images'
 STATIC_AUDIO_PATH = f'{STATIC_PATH}/sounds'
 BCIPY_LOGO_PATH = f'{STATIC_IMAGES_PATH}/gui/cambi.png'
-PREFERENCES_PATH = f'{ROOT}/.bcipy/bcipy_cache'
+PREFERENCES_PATH = f'{ROOT}/bcipy_cache'
 LM_PATH = f'{BCIPY_ROOT}/language/lms'
 SIGNAL_MODEL_FILE_SUFFIX = '.pkl'
 
 # core data configuration
 RAW_DATA_FILENAME = 'raw_data'
+EYE_TRACKER_FILENAME_PREFIX = 'eyetracker_data'
 TRIGGER_FILENAME = 'triggers.txt'
 SESSION_DATA_FILENAME = 'session.json'
 SESSION_SUMMARY_FILENAME = 'session.xlsx'

bcipy/helpers/acquisition.py

@@ -2,17 +2,19 @@
 import logging
 import subprocess
 import time
-from typing import Dict, List, Tuple, Optional
+from typing import Dict, List, Optional, Tuple
 
 import numpy as np
 
+from bcipy.acquisition import (ClientManager, LslAcquisitionClient,
+                               LslDataServer, await_start,
+                               discover_device_spec)
 from bcipy.acquisition.devices import (DeviceSpec, preconfigured_device,
                                        with_content_type)
-from bcipy.acquisition import (LslAcquisitionClient, await_start,
-                               LslDataServer, discover_device_spec,
-                               ClientManager)
+from bcipy.config import BCIPY_ROOT
+from bcipy.config import DEFAULT_DEVICE_SPEC_FILENAME as spec_name
+from bcipy.config import RAW_DATA_FILENAME
 from bcipy.helpers.save import save_device_specs
-from bcipy.config import BCIPY_ROOT, RAW_DATA_FILENAME, DEFAULT_DEVICE_SPEC_FILENAME as spec_name
 
 log = logging.getLogger(__name__)
 
@@ -62,7 +64,7 @@ def init_eeg_acquisition(
             await_start(dataserver)
 
         device_spec = init_device(content_type, device_name)
-        raw_data_name = f'{RAW_DATA_FILENAME}.csv' if content_type == 'EEG' else None
+        raw_data_name = raw_data_filename(device_spec)
 
         client = init_lsl_client(parameters, device_spec, save_folder,
                                  raw_data_name)
@@ -80,6 +82,16 @@ def init_eeg_acquisition(
     return (manager, servers)
 
 
+def raw_data_filename(device_spec: DeviceSpec) -> str:
+    """Returns the name of the raw data file for the given device."""
+    if device_spec.content_type == 'EEG':
+        return f'{RAW_DATA_FILENAME}.csv'
+
+    content_type = '_'.join(device_spec.content_type.split()).lower()
+    name = '_'.join(device_spec.name.split()).lower()
+    return f"{content_type}_data_{name}.csv"
+
+
 def init_device(content_type: str,
                 device_name: Optional[str] = None) -> DeviceSpec:
     """Initialize a DeviceSpec for the given content type.
@@ -206,7 +218,7 @@ def analysis_channels(channels: List[str], device_spec: DeviceSpec) -> list:
     ----------
     - channels(list(str)): list of channel names from the raw_data
     (excluding the timestamp)
-    - device_spec(str): device from which the data was collected
+    - device_spec: device from which the data was collected
 
     Returns
     --------

bcipy/helpers/convert.py

@@ -3,22 +3,20 @@
 import os
 import tarfile
 from pathlib import Path
-from typing import Dict, List, Tuple, Optional
+from typing import Dict, List, Optional, Tuple
 
+import mne
 import numpy as np
-
-from pyedflib import FILETYPE_EDFPLUS, EdfWriter, FILETYPE_BDFPLUS
+from mne.io import RawArray
+from pyedflib import FILETYPE_BDFPLUS, FILETYPE_EDFPLUS, EdfWriter
 from tqdm import tqdm
 
-from bcipy.config import RAW_DATA_FILENAME, TRIGGER_FILENAME, DEFAULT_PARAMETER_FILENAME
+from bcipy.config import (DEFAULT_PARAMETER_FILENAME, RAW_DATA_FILENAME,
+                          TRIGGER_FILENAME)
 from bcipy.helpers.load import load_json_parameters, load_raw_data
 from bcipy.helpers.raw_data import RawData
-from bcipy.signal.process import Composition, get_default_transform
 from bcipy.helpers.triggers import trigger_decoder, trigger_durations
-
-import mne
-from mne.io import RawArray
-
+from bcipy.signal.process import Composition, get_default_transform
 
 logger = logging.getLogger(__name__)
 

bcipy/helpers/demo/demo_visualization.py

@@ -13,19 +13,16 @@
     - - `python bcipy/helpers/demo/demo_visualization.py --save"`
         this will save the visualizations generated to the provided or selected path
 """
-from bcipy.config import (TRIGGER_FILENAME, DEFAULT_PARAMETER_FILENAME,
-                          RAW_DATA_FILENAME, DEFAULT_DEVICE_SPEC_FILENAME)
+import bcipy.acquisition.devices as devices
+from bcipy.config import (DEFAULT_DEVICE_SPEC_FILENAME,
+                          DEFAULT_PARAMETER_FILENAME, RAW_DATA_FILENAME,
+                          TRIGGER_FILENAME)
 from bcipy.helpers.acquisition import analysis_channels
-from bcipy.signal.process import get_default_transform
-from bcipy.helpers.load import (
-    load_experimental_data,
-    load_json_parameters,
-    load_raw_data,
-)
-from bcipy.helpers.visualization import visualize_erp
+from bcipy.helpers.load import (load_experimental_data, load_json_parameters,
+                                load_raw_data)
 from bcipy.helpers.triggers import TriggerType, trigger_decoder
-import bcipy.acquisition.devices as devices
-
+from bcipy.helpers.visualization import visualize_erp
+from bcipy.signal.process import get_default_transform
 
 if __name__ == '__main__':
     import argparse

bcipy/helpers/raw_data.py

@@ -181,6 +181,12 @@ def append(self, row: List):
         self._rows.append(row)
         self._dataframe = None
 
+    def __str__(self) -> str:
+        return f"RawData({self.daq_type})"
+
+    def __repr__(self) -> str:
+        return f"RawData({self.daq_type})"
+
 
 def maybe_float(val):
     """Attempt to convert the given value to float. If conversion fails return
@@ -323,7 +329,7 @@ def load(filename: str) -> RawData:
 
 
 def read_metadata(file_obj: TextIO) -> Tuple[str, float]:
-    """Reads the metadata from an open raw data file and retuns the result as
+    """Reads the metadata from an open raw data file and returns the result as
     a tuple. Increments the reader.
 
     Parameters

bcipy/helpers/stimuli.py

@@ -14,6 +14,7 @@
 
 from bcipy.helpers.exceptions import BciPyCoreException
 from bcipy.helpers.list import grouper
+from bcipy.helpers.symbols import alphabet
 
 # Prevents pillow from filling the console with debug info
 logging.getLogger('PIL').setLevel(logging.WARNING)
@@ -129,7 +130,7 @@ def __call__(self,
         Returns:
             reshaped_data (np.ndarray): inquiry data of shape (Channels, Inquiries, Samples)
             labels (np.ndarray): integer label for each inquiry. With `trials_per_inquiry=K`,
-                a label of [0, K-1] indicates the position of `target_label`, or label of K indicates
+                a label of [0, K-1] indicates the position of `target_label`, or label of [0 ... 0] indicates
                 `target_label` was not present.
             reshaped_trigger_timing (List[List[int]]): For each inquiry, a list of the sample index where each trial
                 begins, accounting for the prestim buffer that may have been added to the front of each inquiry.
@@ -229,6 +230,114 @@ def extract_trials(
         return np.stack(new_trials, 1)  # C x T x S
 
 
+class GazeReshaper:
+    def __call__(self,
+                 inq_start_times: List[float],
+                 target_symbols: List[str],
+                 gaze_data: np.ndarray,
+                 sample_rate: int,
+                 channel_map: List[int] = None,
+                 ) -> dict:
+        """Extract inquiry data and labels. Different from the EEG inquiry, the gaze inquiry window starts with
+        the first flicker and ends with the last flicker in the inquiry. Each inquiry has a length of ~3 seconds.
+        The labels are provided in the target_symbols list. It returns a Dict, where keys are the target symbols and
+        the values are inquiries (appended in order of appearance) where the corresponding target symbol is prompted.
+        Optional outputs:
+        reshape_data is the list of data reshaped into (Inquiries, Channels, Samples), where inquirires are appended
+        in chronological order. labels returns the list of target symbols in each inquiry.
+
+        Args:
+            inq_start_times (List[float]): Timestamp of each event in seconds
+            target_symbols (List[str]): Prompted symbol in each inquiry
+            gaze_data (np.ndarray): shape (channels, samples) eye tracking data
+            sample_rate (int): sample rate of data provided in eeg_data
+            channel_map (List[int], optional): Describes which channels to include or discard.
+                Defaults to None; all channels will be used.
+
+        Returns:
+            data_by_targets (dict): Dictionary where keys are the symbol set and values are the appended inquiries
+            for each symbol. dict[Key] = (np.ndarray) of shape (Channels, Samples)
+
+            reshaped_data (List[float]) [optional]: inquiry data of shape (Inquiries, Channels, Samples)
+            labels (List[str]) [optional] : Target symbol in each inquiry.
+        """
+        if channel_map:
+            # Remove the channels that we are not interested in
+            channels_to_remove = [idx for idx, value in enumerate(channel_map) if value == 0]
+            gaze_data = np.delete(gaze_data, channels_to_remove, axis=0)
+
+        # Find the value closest to (& greater than) inq_start_times
+        gaze_data_timing = gaze_data[-1, :].tolist()
+
+        start_times = []
+        for times in inq_start_times:
+            temp = list(filter(lambda x: x > times, gaze_data_timing))
+            if len(temp) > 0:
+                start_times.append(temp[0])
+
+        triggers = []
+        for val in start_times:
+            triggers.append(gaze_data_timing.index(val))
+
+        # Label for every inquiry
+        labels = target_symbols
+
+        symbol_set = alphabet()
+
+        # Create a dictionary with symbols as keys and data as values
+        # 'A': [], 'B': [] ...
+        data_by_targets = {}
+        for symbol in symbol_set:
+            data_by_targets[symbol] = []
+
+        window_length = 3  # seconds, total length of flickering after prompt for each inquiry
+
+        reshaped_data = []
+        # Merge the inquiries if they have the same target letter:
+        for i, inquiry_index in enumerate(triggers):
+            start = inquiry_index
+            stop = int(inquiry_index + (sample_rate * window_length))   # (60 samples * 3 seconds)
+            # Check if the data exists for the inquiry:
+            if stop > len(gaze_data[0, :]):
+                continue
+
+            reshaped_data.append(gaze_data[:, start:stop])
+            # (Optional) extracted data (Inquiries x Channels x Samples)
+
+            # Populate the dict by appending the inquiry to the correct key:
+            data_by_targets[labels[i]].append(gaze_data[:, start:stop])
+
+        # After populating, flatten the arrays in the dictionary to (Channels x Samples):
+        for symbol in symbol_set:
+            if len(data_by_targets[symbol]) > 0:
+                data_by_targets[symbol] = np.transpose(np.array(data_by_targets[symbol]), (1, 0, 2))
+                data_by_targets[symbol] = np.reshape(data_by_targets[symbol], (len(data_by_targets[symbol]), -1))
+
+            # Note that this is a workaround to the issue of having different number of targetness in
+            # each symbol. If a target symbol is prompted more than once, the data is appended to the dict as a list.
+            # Which is why we need to convert it to a (np.ndarray) and flatten the dimensions.
+            # This is not ideal, but it works for now.
+
+        # return np.stack(reshaped_data, 0), labels
+        return data_by_targets
+
+    @staticmethod
+    def centralize_all_data(data, symbol_pos):
+        """ Using the symbol locations in matrix, centralize all data (in Tobii units).
+        This data will only be used in certain model types.
+        Args:
+            data (np.ndarray): Data in shape of num_channels x num_samples
+            symbol_pos (np.ndarray(float)): Array of the current symbol posiiton in Tobii units
+        Returns:
+            data (np.ndarray): Centralized data in shape of num_channels x num_samples
+        """
+        for i in range(len(data)):
+            data[i] = data[i] - symbol_pos
+            # TODO: List comprehension
+
+        return data
+
+
 class TrialReshaper(Reshaper):
     def __call__(self,
                  trial_targetness_label: list,

bcipy/helpers/system_utils.py

@@ -211,7 +211,9 @@ def configure_logger(
         '[%(threadName)-9s][%(asctime)s][%(name)s][%(levelname)s]: %(message)s'))
     root_logger.addHandler(handler)
 
-    print(f'Printing all BciPy logs to: {logfile}')
+    # print to console the absolute path of the log file to aid in debugging
+    path_to_logs = os.path.abspath(logfile)
+    print(f'Printing all BciPy logs to: {path_to_logs}')
 
     if version:
         logging.info(f'Start of Session for BciPy Version: ({version})')

bcipy/helpers/tests/test_acquisition.py

@@ -4,10 +4,13 @@
 from pathlib import Path
 from unittest.mock import Mock, patch
 
+from bcipy.acquisition.devices import DeviceSpec
 from bcipy.config import DEFAULT_PARAMETERS_PATH
-from bcipy.helpers.acquisition import (init_device, init_eeg_acquisition,
+from bcipy.helpers.acquisition import (RAW_DATA_FILENAME, init_device,
+                                       init_eeg_acquisition,
                                        max_inquiry_duration, parse_stream_type,
-                                       server_spec, stream_types)
+                                       raw_data_filename, server_spec,
+                                       stream_types)
 from bcipy.helpers.load import load_json_parameters
 from bcipy.helpers.save import init_save_data_structure
 
@@ -166,6 +169,24 @@ def test_server_spec_without_named_device(self, with_content_type_mock):
         with_content_type_mock.assert_called_with('EEG')
         self.assertEqual(device_spec, device1)
 
+    def test_raw_data_filename_eeg(self):
+        """Test generation of filename for EEG devices"""
+        device = DeviceSpec(name='DSI-24',
+                            channels=[],
+                            sample_rate=300.0,
+                            content_type='EEG')
+        self.assertEqual(raw_data_filename(device), f'{RAW_DATA_FILENAME}.csv')
+
+    def test_raw_data_filename(self):
+        """Test generation of filename"""
+
+        device = DeviceSpec(name='Tobii-P0',
+                            channels=[],
+                            sample_rate=60,
+                            content_type='EYETRACKER')
+        self.assertEqual(raw_data_filename(device),
+                         'eyetracker_data_tobii-p0.csv')
+
 
 if __name__ == '__main__':
     unittest.main()