LM Toolkit Refactor

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, 3.10.6]
+        python-version: [3.9, 3.10.6]
 
     steps:
     - uses: actions/checkout@v2
@@ -69,7 +69,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, 3.10.6]
+        python-version: [3.9, 3.10.6]
 
     steps:
     - uses: actions/checkout@v2
@@ -93,9 +93,6 @@ jobs:
     - name: lint
       run: |
         make lint
-    - name: integration-test
-      run: |
-        make integration-test
     - name: build
       run: |
         make build
@@ -106,7 +103,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, 3.10.6]
+        python-version: [3.9, 3.10.6]
 
     steps:
     - uses: actions/checkout@v4
@@ -133,9 +130,6 @@ jobs:
     - name: lint
       run: |
         make lint
-    - name: integration-test
-      run: |
-        make integration-test
     - name: build
       run: |
         make build

Makefile

@@ -13,7 +13,6 @@ test-all:
 	make coverage-report
 	make type
 	make lint
-	make integration-test
 
 unit-test:
 	pytest --mpl -k "not slow"

bcipy/core/tests/resources/mock_session/parameters.json

@@ -680,7 +680,7 @@
     "recommended": [
       "UNIFORM",
       "CAUSAL",
-      "KENLM",
+      "NGRAM",
       "MIXTURE",
       "ORACLE"
     ],

bcipy/exceptions.py

@@ -90,13 +90,6 @@ class TaskConfigurationException(BciPyCoreException):
     ...
 
 
-class InvalidLanguageModelException(BciPyCoreException):
-    """Invalid Language Model Exception.
-
-    Thrown when attempting to load a language model from an invalid path"""
-    ...
-
-
 class KenLMInstallationException(BciPyCoreException):
     """KenLM Installation Exception.
 

bcipy/helpers/copy_phrase_wrapper.py

@@ -10,7 +10,7 @@
 from bcipy.core.symbols import BACKSPACE_CHAR
 from bcipy.exceptions import BciPyCoreException
 from bcipy.helpers.language_model import histogram, with_min_prob
-from bcipy.language.main import LanguageModel
+from bcipy.language.main import LanguageModelAdapter
 from bcipy.task.control.criteria import (CriteriaEvaluator,
                                          MaxIterationsCriteria,
                                          MinIterationsCriteria,
@@ -58,7 +58,7 @@ class CopyPhraseWrapper:
     def __init__(self,
                  min_num_inq: int,
                  max_num_inq: int,
-                 lmodel: LanguageModel,
+                 lmodel: LanguageModelAdapter,
                  alp: List[str],
                  evidence_names: List[EvidenceType] = [
                      EvidenceType.LM, EvidenceType.ERP

bcipy/helpers/language_model.py

@@ -6,29 +6,29 @@
 import numpy as np
 
 from bcipy.core.symbols import alphabet
-from bcipy.language.main import LanguageModel, ResponseType
+from bcipy.language.main import LanguageModelAdapter, ResponseType
 
 # pylint: disable=unused-import
 # flake8: noqa
 
 """Only imported models will be included in language_models_by_name"""
 # flake8: noqa
-from bcipy.exceptions import InvalidLanguageModelException
-from bcipy.language.model.causal import CausalLanguageModel
-from bcipy.language.model.kenlm import KenLMLanguageModel
-from bcipy.language.model.mixture import MixtureLanguageModel
-from bcipy.language.model.oracle import OracleLanguageModel
-from bcipy.language.model.uniform import UniformLanguageModel
+from bcipy.language.model.causal import CausalLanguageModelAdapter
+from bcipy.language.model.ngram import NGramLanguageModelAdapter
+from bcipy.language.model.mixture import MixtureLanguageModelAdapter
+from bcipy.language.model.oracle import OracleLanguageModelAdapter
+from bcipy.language.model.uniform import UniformLanguageModelAdapter
 
 
-def language_models_by_name() -> Dict[str, LanguageModel]:
+def language_models_by_name() -> Dict[str, LanguageModelAdapter]:
     """Returns available language models indexed by name."""
-    return {lm.name(): lm for lm in LanguageModel.__subclasses__()}
+    return {lm.name(): lm for lm in LanguageModelAdapter.__subclasses__()}
 
 
-def init_language_model(parameters: dict) -> LanguageModel:
+def init_language_model(parameters: dict) -> LanguageModelAdapter:
     """
-    Init Language Model configured in the parameters.
+    Init Language Model configured in the parameters. If no language model is
+    specified, a uniform language model is returned.
 
     Parameters
     ----------
@@ -37,7 +37,7 @@ def init_language_model(parameters: dict) -> LanguageModel:
 
     Returns
     -------
-        instance of a LanguageModel
+        instance of a LanguageModelAdapter
     """
 
     language_models = language_models_by_name()
@@ -48,9 +48,11 @@ def init_language_model(parameters: dict) -> LanguageModel:
 
     # select the relevant parameters into a dict.
     params = {key: parameters[key] for key in args & parameters.keys()}
-    return model(response_type=ResponseType.SYMBOL,
-                 symbol_set=alphabet(parameters),
-                 **params)
+
+    return model(
+        response_type=ResponseType.SYMBOL,
+        symbol_set=alphabet(parameters),
+        **params)
 
 
 def norm_domain(priors: List[Tuple[str, float]]) -> List[Tuple[str, float]]:

bcipy/helpers/tests/test_copy_phrase_wrapper.py

@@ -1,21 +1,20 @@
 import unittest
 
 from bcipy.helpers.copy_phrase_wrapper import CopyPhraseWrapper
-from bcipy.core.symbols import alphabet
-from bcipy.language.model.uniform import UniformLanguageModel
+from bcipy.core.symbols import DEFAULT_SYMBOL_SET
+from bcipy.language.model.uniform import UniformLanguageModelAdapter
 from bcipy.task.data import EvidenceType
 
 
 class TestCopyPhraseWrapper(unittest.TestCase):
     """Test CopyPhraseWrapper"""
 
     def test_valid_letters(self):
-        alp = alphabet()
         cp = CopyPhraseWrapper(
             min_num_inq=1,
             max_num_inq=50,
             lmodel=None,
-            alp=alp,
+            alp=DEFAULT_SYMBOL_SET,
             task_list=[("HELLO_WORLD", "HE")],
             is_txt_stim=True,
             evidence_names=[EvidenceType.LM, EvidenceType.ERP],
@@ -104,13 +103,12 @@ def test_valid_letters(self):
                            ["nontarget", "nontarget"])
 
     def test_init_series(self):
-        alp = alphabet()
 
         copy_phrase_task = CopyPhraseWrapper(
             min_num_inq=1,
             max_num_inq=50,
-            lmodel=UniformLanguageModel(symbol_set=alp),
-            alp=alp,
+            lmodel=UniformLanguageModelAdapter(symbol_set=DEFAULT_SYMBOL_SET),
+            alp=DEFAULT_SYMBOL_SET,
             task_list=[("HELLO_WORLD", "HE")],
             is_txt_stim=True,
             evidence_names=[EvidenceType.LM, EvidenceType.ERP],

bcipy/language/README.md

@@ -1,15 +1,13 @@
 # Language
 
-BciPy Language module provides an interface for word and character level predictions.
+BciPy Language module provides an interface for word and character level predictions. This module primarily relies upon the AAC-TextPredict package (aactextpredict on PyPI) for its probability calculations. More information on this package can be found on our [GitHub repo](https://github.com/kdv123/textpredict)
 
-The core methods of any `LanguageModel` include:
+The core methods of any `LanguageModelAdapter` include:
 
 > `predict` - given typing evidence input, return a prediction (character or word).
 
 > `load` - load a pre-trained model given a path (currently BciPy does not support training language models!)
 
-> `update` - update internal state of your model.
-
 You may of course define other methods, however all integrated BciPy experiments using your model will require those to be defined!
 
 The language module has the following structure:
@@ -18,7 +16,7 @@ The language module has the following structure:
 
 > `lms` - The default location for the model resources.
 
-> `model` - The python classes for each LanguageModel subclass. Detailed descriptions of each can be found below.
+> `model` - The python classes for each LanguageModelAdapter subclass. Detailed descriptions of each can be found below.
 
 > `sets` - Different phrase sets that can be used to test the language model classes.
 
@@ -28,22 +26,22 @@ The language module has the following structure:
 
 ## Uniform Model
 
-The UniformLanguageModel provides equal probabilities for all symbols in the symbol set. This model is useful for evaluating other aspects of the system, such as EEG signal quality, without any influence from a language model.
+The UniformLanguageModelAdapter provides equal probabilities for all symbols in the symbol set. This model is useful for evaluating other aspects of the system, such as EEG signal quality, without any influence from a language model.
 
-## KenLM Model
-The KenLMLanguageModel utilizes a pretrained n-gram language model to generate probabilities for all symbols in the symbol set. N-gram models use frequencies of different character sequences to generate their predictions. Models trained on AAC-like data can be found [here](https://imagineville.org/software/lm/dec19_char/). For faster load times, it is recommended to use the binary models located at the bottom of the page. The default parameters file utilizes `lm_dec19_char_large_12gram.kenlm`. If you have issues accessing, please reach out to us on GitHub or via email at `cambi_support@googlegroups.com`.
+## NGram Model
+The NGramLanguageModelAdapter utilizes a pretrained n-gram language model to generate probabilities for all symbols in the symbol set. N-gram models use frequencies of different character sequences to generate their predictions. Models trained on AAC-like data can be found [here](https://imagineville.org/software/lm/dec19_char/). For faster load times, it is recommended to use the binary models located at the bottom of the page. The default parameters file utilizes `lm_dec19_char_large_12gram.kenlm`. If you have issues accessing, please reach out to us on GitHub or via email at `cambi_support@googlegroups.com`.
 
 For models that import the kenlm module, this must be manually installed using `pip install kenlm==0.1 --global-option="max_order=12"`.
 
 ## Causal Model
-The CausalLanguageModel class can use any causal language model from Huggingface, though it has only been tested with gpt2, facebook/opt, and distilgpt2 families of models. Causal language models predict the next token in a sequence of tokens. For the many of these models, byte-pair encoding (BPE) is used for tokenization. The main idea of BPE is to create a fixed-size vocabulary that contains common English subword units. Then a less common word would be broken down into several subword units in the vocabulary. For example, the tokenization of character sequence `peanut_butter_and_jel` would be:
+The CausalLanguageModelAdapter class can use any causal language model from Huggingface, though it has only been tested with gpt2, facebook/opt, and distilgpt2 families of models (including the domain-adapted figmtu/opt-350m-aac). Causal language models predict the next token in a sequence of tokens. For the many of these models, byte-pair encoding (BPE) is used for tokenization. The main idea of BPE is to create a fixed-size vocabulary that contains common English subword units. Then a less common word would be broken down into several subword units in the vocabulary. For example, the tokenization of character sequence `peanut_butter_and_jel` would be:
 > *['pe', 'anut', '_butter', '_and', '_j', 'el']*
 
-Therefore, in order to generate a predictive distribution on the next character, we need to examine all the possibilities that could complete the final subword tokens in the input sequences. We must remove at least one token from the end of the context to allow the model the option of extending it, as opposed to only adding a new token. Removing more tokens allows the model more flexibility and may lead to better predictions, but at the cost of a higher prediction time. In this model we remove all of the subword tokens in the current (partially-typed) word to allow it the most flexibility. We then ask the model to estimate the likelihood of the next token and evaluate each token that matches our context. For efficiency, we only track a certain number of hypotheses at a time, known as the beam width, and each hypothesis until it surpasses the context. We can then store the likelihood for each final prediction in a list based on the character that directly follows the context. Once we have no more hypotheses to extend, we can sum the likelihoods stored for each character in our symbol set and normalize so they sum to 1, giving us our final distribution.
+Therefore, in order to generate a predictive distribution on the next character, we need to examine all the possibilities that could complete the final subword tokens in the input sequences. We must remove at least one token from the end of the context to allow the model the option of extending it, as opposed to only adding a new token. Removing more tokens allows the model more flexibility and may lead to better predictions, but at the cost of a higher prediction time. In this model we remove all of the subword tokens in the current (partially-typed) word to allow it the most flexibility. We then ask the model to estimate the likelihood of the next token and evaluate each token that matches our context. For efficiency, we only track a certain number of hypotheses at a time, known as the beam width, and each hypothesis until it surpasses the context. We can then store the likelihood for each final prediction in a list based on the character that directly follows the context. Once we have no more hypotheses to extend, we can sum the likelihoods stored for each character in our symbol set and normalize so they sum to 1, giving us our final distribution. More details on this process can be found in our paper, [Adapting Large Language Models for Character-based Augmentative and Alternative Communication](https://arxiv.org/abs/2501.10582).
 
 
 ## Mixture Model
-The MixtureLanguageModel class allows for the combination of two or more supported models. The selected models are mixed according to the provided weights, which can be tuned using the Bcipy/scripts/python/mixture_tuning.py script. It is not recommended to use more than one "heavy-weight" model with long prediction times (the CausalLanguageModel) since this model will query each component model and parallelization is not currently supported.
+The MixtureLanguageModelAdapter class allows for the combination of two or more supported models. The selected models are mixed according to the provided weights, which can be tuned using the Bcipy/scripts/python/mixture_tuning.py script. It is not recommended to use more than one "heavy-weight" model with long prediction times (the CausalLanguageModel) since this model will query each component model and parallelization is not currently supported.
 
 # Contact Information
 

bcipy/language/__init__.py

@@ -1,6 +1,6 @@
-from .main import LanguageModel, ResponseType
+from .main import LanguageModelAdapter, ResponseType
 
 __all__ = [
-    "LanguageModel",
+    "LanguageModelAdapter",
     "ResponseType",
 ]

bcipy/language/demo/demo_causal.py

@@ -1,26 +1,36 @@
-from bcipy.language.model.causal import CausalLanguageModel
-from bcipy.core.symbols import alphabet
+from bcipy.language.model.causal import CausalLanguageModelAdapter
+from bcipy.core.symbols import DEFAULT_SYMBOL_SET
 from bcipy.language.main import ResponseType
 
 
 if __name__ == "__main__":
-    symbol_set = alphabet()
     response_type = ResponseType.SYMBOL
-    lm = CausalLanguageModel(response_type, symbol_set, lang_model_name="gpt2")
+    lm = CausalLanguageModelAdapter(response_type, DEFAULT_SYMBOL_SET, lang_model_name="figmtu/opt-350m-aac")
 
-    next_char_pred = lm.state_update(list("does_it_make_sen"))
-    print(next_char_pred)
+    print("Target sentence: does_it_make_sense\n")
+
+    next_char_pred = lm.predict(list("does_it_make_sen"))
+    print(f"Context: does_it_make_sen")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("S") + 1
-    print(correct_char_rank)
-    next_char_pred = lm.state_update(list("does_it_make_sens"))
-    print(next_char_pred)
+    print(f"Correct character rank: {correct_char_rank}\n")
+
+    next_char_pred = lm.predict(list("does_it_make_sens"))
+    print(f"Context: does_it_make_sens")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("E") + 1
-    print(correct_char_rank)
-    next_char_pred = lm.state_update(list("does_it_make_sense"))
-    print(next_char_pred)
+    print(f"Correct character rank: {correct_char_rank}\n")
+
+    next_char_pred = lm.predict(list("does_it_make_sense"))
+    print(f"Context: does_it_make_sense")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("_") + 1
-    print(correct_char_rank)
-    next_char_pred = lm.state_update(list("i_like_zebra"))
-    print(next_char_pred)
+    print(f"Correct character rank: {correct_char_rank}\n")
+
+    print("Target sentence: i_like_zebras\n")
+
+    next_char_pred = lm.predict(list("i_like_zebra"))
+    print(f"Context: i_like_zebra")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("S") + 1
-    print(correct_char_rank)
+    print(f"Correct character rank: {correct_char_rank}\n")

bcipy/language/demo/demo_mixture.py

@@ -1,22 +1,37 @@
-from bcipy.language.model.mixture import MixtureLanguageModel
-from bcipy.core.symbols import alphabet
+from bcipy.language.model.mixture import MixtureLanguageModelAdapter
+from bcipy.core.symbols import DEFAULT_SYMBOL_SET
 from bcipy.language.main import ResponseType
 
 
 if __name__ == "__main__":
-    symbol_set = alphabet()
     response_type = ResponseType.SYMBOL
-    lm = MixtureLanguageModel(response_type, symbol_set)
+    # Load the default mixture model from lm_params.json
+    lm = MixtureLanguageModelAdapter(response_type, DEFAULT_SYMBOL_SET)
 
-    next_char_pred = lm.state_update(list("does_it_make_sen"))
-    print(next_char_pred)
+    print("Target sentence: does_it_make_sense\n")
+
+    next_char_pred = lm.predict(list("does_it_make_sen"))
+    print(f"Context: does_it_make_sen")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("S") + 1
-    print(correct_char_rank)
-    next_char_pred = lm.state_update(list("does_it_make_sens"))
-    print(next_char_pred)
+    print(f"Correct character rank: {correct_char_rank}\n")
+
+    next_char_pred = lm.predict(list("does_it_make_sens"))
+    print(f"Context: does_it_make_sens")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("E") + 1
-    print(correct_char_rank)
-    next_char_pred = lm.state_update(list("does_it_make_sense"))
-    print(next_char_pred)
+    print(f"Correct character rank: {correct_char_rank}\n")
+
+    next_char_pred = lm.predict(list("does_it_make_sense"))
+    print(f"Context: does_it_make_sense")
+    print(f"Predictions: {next_char_pred}")
     correct_char_rank = [c[0] for c in next_char_pred].index("_") + 1
-    print(correct_char_rank)
+    print(f"Correct character rank: {correct_char_rank}\n")
+
+    print("Target sentence: i_like_zebras\n")
+
+    next_char_pred = lm.predict(list("i_like_zebra"))
+    print(f"Context: i_like_zebra")
+    print(f"Predictions: {next_char_pred}")
+    correct_char_rank = [c[0] for c in next_char_pred].index("S") + 1
+    print(f"Correct character rank: {correct_char_rank}\n")