fix: set TOKENIZERS_PARALLELISM env var to false to get rid of warning

ErikBjare · Dec 8, 2024 · 75b298b · 75b298b
1 parent 696e722
commit 75b298b
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/gptme/init.py b/gptme/init.py
@@ -1,5 +1,6 @@
 import atexit
 import logging
+import os
 from typing import cast
 
 from dotenv import load_dotenv
@@ -32,6 +33,10 @@ def init(model: str | None, interactive: bool, tool_allowlist: list[str] | None)
     logger.debug("Started")
     load_dotenv()
 
+    # fixes issues with transformers parallelism
+    # which also leads to "Context leak detected, msgtracer returned -1"
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
     config = get_config()
 
     # get from config