remove langchain (#42)

* add log mode, refine doc

* fix react

* remove langchain

---------

Co-authored-by: Satan <liuxk2019@mail.sustech.edu.cn>

Author: liuxukun2000

gentopia/tools/calculator.py

@@ -1,12 +1,31 @@
 from typing import AnyStr
-
-from langchain import OpenAI, LLMMathChain
-
+import re
+import numexpr
 from .basetool import *
+import math
 
 class CalculatorArgs(BaseModel):
     expression: str = Field(..., description="A mathematical expression.")
 
+def _evaluate_expression(expression: str) -> str:
+    try:
+        local_dict = {"pi": math.pi, "e": math.e}
+        output = str(
+            numexpr.evaluate(
+                expression.strip(),
+                global_dict={},  # restrict access to globals
+                local_dict=local_dict,  # add common mathematical functions
+            )
+        )
+    except Exception as e:
+        raise ValueError(
+            f'LLMMathChain._evaluate("{expression}") raised error: {e}.'
+            " Please try again with a valid numerical expression"
+        )
+
+    # Remove any leading and trailing brackets from the output
+    return re.sub(r"^\[|\]$", "", output)
+
 class Calculator(BaseTool):
     """docstring for Calculator"""
     name = "calculator"
@@ -15,10 +34,9 @@ class Calculator(BaseTool):
     args_schema: Optional[Type[BaseModel]] = CalculatorArgs
 
     def _run(self, expression: AnyStr) -> Any:
-        llm = OpenAI(temperature=0)
-        tool = LLMMathChain(llm=llm, verbose=self.verbose)
-        response = tool(expression)
-        evidence = response["answer"].replace("Answer:", "").strip()
+
+        response = _evaluate_expression(expression)
+        evidence = response.strip()
         return evidence
 
     async def _arun(self, *args: Any, **kwargs: Any) -> Any:

gentopia/tools/gradio_tools/tools/gradio_tool.py

@@ -9,14 +9,6 @@
 from gradio_client.client import Job
 from gradio_client.utils import QueueError
 
-try:
-    import langchain as lc
-
-    LANGCHAIN_INSTALLED = True
-except (ModuleNotFoundError, ImportError):
-    LANGCHAIN_INSTALLED = False
-
-
 class GradioTool:
     def __init__(
         self,
@@ -110,17 +102,5 @@ def block(self):
             self._block = gr.load(name=self.src, src="spaces")
         return self._block
 
-    # Optional langchain functionalities
-    @property
-    def langchain(self) -> "langchain.agents.Tool":  # type: ignore
-        if not LANGCHAIN_INSTALLED:
-            raise ModuleNotFoundError(
-                "langchain must be installed to access langchain tool"
-            )
-
-        return lc.agents.Tool(  # type: ignore
-            name=self.name, func=self.run, description=self.description
-        )
-
     def __repr__(self) -> str:
         return f"GradioTool(name={self.name}, src={self.src})"

gentopia/tools/search_doc.py

@@ -1,9 +1,7 @@
 from typing import AnyStr
-
-from langchain.document_loaders import TextLoader
-from langchain.indexes import VectorstoreIndexCreator
-
 from .basetool import *
+from .utils.document_loaders.text_loader import TextLoader
+from .utils.vector_store import VectorstoreIndexCreator
 
 
 class SearchDoc(BaseTool):
@@ -16,7 +14,7 @@ class SearchDoc(BaseTool):
 
     def _run(self, query: AnyStr) -> AnyStr:
         loader = TextLoader(self.doc_path)
-        vector_store = VectorstoreIndexCreator().from_loaders([loader]).vectorstore
+        vector_store = VectorstoreIndexCreator().from_loaders([loader])
         evidence = vector_store.similarity_search(query, k=1)[0].page_content
         return evidence
 

gentopia/tools/utils/document_loaders/__init__.py

@@ -0,0 +1,44 @@
+from abc import abstractmethod, ABC
+from typing import Optional, List, Iterator, Sequence, Any, Callable
+
+from gentopia.tools.utils import Document
+from gentopia.tools.utils.document_loaders.text_splitter import TextSplitter, RecursiveCharacterTextSplitter
+
+
+class BaseLoader(ABC):
+    """Interface for loading documents.
+
+    Implementations should implement the lazy-loading method using generators
+    to avoid loading all documents into memory at once.
+
+    The `load` method will remain as is for backwards compatibility, but it's
+    implementation should be just `list(self.lazy_load())`.
+    """
+
+    # Sub-classes should implement this method
+    # as return list(self.lazy_load()).
+    # This method returns a List which is materialized in memory.
+    @abstractmethod
+    def load(self) -> List[Document]:
+        """Load data into document objects."""
+
+    def load_and_split(
+        self, text_splitter: Optional[TextSplitter] = None
+    ) -> List[Document]:
+        """Load documents and split into chunks."""
+        if text_splitter is None:
+            _text_splitter: TextSplitter = RecursiveCharacterTextSplitter()
+        else:
+            _text_splitter = text_splitter
+        docs = self.load()
+        return _text_splitter.split_documents(docs)
+
+    # Attention: This method will be upgraded into an abstractmethod once it's
+    #            implemented in all the existing subclasses.
+    def lazy_load(
+        self,
+    ) -> Iterator[Document]:
+        """A lazy loader for document content."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement lazy_load()"
+        )

gentopia/tools/utils/document_loaders/base_loader.py

@@ -2,8 +2,10 @@
 from typing import Optional, List, Iterator, Sequence, Any, Callable
 
 from gentopia.tools.utils import Document
+from gentopia.tools.utils.document_loaders.base_loader import BaseLoader
 
-class TextLoader:
+
+class TextLoader(BaseLoader):
     """Load text files."""
 
     def __init__(self, file_path: str, encoding: Optional[str] = None):