removed dict info and put in another file

Author: VinciGit00

scrapegraphai/helpers/__init__.py

@@ -4,3 +4,4 @@
 """
 from .nodes_metadata import nodes_metadata
 from .schemas import graph_schema
+from .models_tokens import models_tokens

scrapegraphai/helpers/models_tokens.py

@@ -0,0 +1,17 @@
+"""
+Models token 
+"""
+models_tokens = {
+    "gpt-3.5-turbo-0125": 16385,
+    "gpt-3.5-turbo": 4096,
+    "gpt-3.5-turbo-1106": 16385,
+    "gpt-3.5-turbo-instruct": 4096,
+    "gpt-4-0125-preview": 128000,
+    "gpt-4-turbo-preview": 128000,
+    "gpt-4-1106-preview": 128000,
+    "gpt-4-vision-preview": 128000,
+    "gpt-4": 8192,
+    "gpt-4-0613": 8192,
+    "gpt-4-32k": 32768,
+    "gpt-4-32k-0613": 32768,
+}

scrapegraphai/utils/token_calculator.py

@@ -3,21 +3,8 @@
 """
 from typing import List
 import tiktoken
+from ..helpers.models_tokens import models_tokens
 
-models_tokens = {
-    "gpt-3.5-turbo-0125": 16385,
-    "gpt-3.5-turbo": 4096,
-    "gpt-3.5-turbo-1106": 16385,
-    "gpt-3.5-turbo-instruct": 4096,
-    "gpt-4-0125-preview": 128000,
-    "gpt-4-turbo-preview": 128000,
-    "gpt-4-1106-preview": 128000,
-    "gpt-4-vision-preview": 128000,
-    "gpt-4": 8192,
-    "gpt-4-0613": 8192,
-    "gpt-4-32k": 32768,
-    "gpt-4-32k-0613": 32768,
-}
 
 def truncate_text_tokens(text: str, model: str, encoding_name: str) -> List[str]:
     """