add base_loader abstraction and update base_io

camel/loaders/base_io.py

@@ -21,6 +21,41 @@
 
 from camel.utils import dependencies_required
 
+from .base_loader import BaseLoader
+
+
+class BaseIOLoader(BaseLoader):
+    def __init__(self, config):
+        super().__init__(config)
+
+    def load(self, source, **kwargs) -> "File":
+        r"""Load files from different source types.
+
+        Args:
+            source: A source object that respresent the contents of the file.
+            The format can be BytesIO, raw bytes or text with diffent
+            extentions.
+            **kwargs: Additional keyword argments.
+
+        Return:
+            File: A File object.
+        """
+        if isinstance(source, BytesIO):
+            file_obj = source
+        elif isinstance(source, bytes):
+            file_obj = BytesIO(source)
+        elif isinstance(source, str):
+            # If a string is provided, assume it's a file path and read
+            # the file's binary content.
+            with open(source, 'rb') as f:
+                file_obj = BytesIO(f.read())
+        else:
+            raise ValueError("Unsupported source type.")
+        # Extract the filename from kwargs; default to 'unknown' if
+        # filename is not provided.
+        filename = kwargs.get("filename", "unknown")
+        return create_file(file_obj, filename)
+
 
 def create_file(file: BytesIO, filename: str) -> "File":
     r"""Reads an uploaded file and returns a File object.

camel/loaders/base_loader.py

@@ -0,0 +1,47 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from abc import ABC, abstractmethod
+
+
+class BaseLoader(ABC):
+    def __init__(self, config):
+        r"""Common configuration initialization.
+        Args:
+            config: A configuration object or dictionary.
+        """
+        self.config = config
+
+    @abstractmethod
+    def load(self, source, **kwargs):
+        r"""Main abstract method for loading data.
+        Args:
+            source: The data source (e.g., file path, URL).
+            **kwargs: Additional keyword arguments for loading data.
+
+        Returns:
+            Loaded data.
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def supported_formats(self):
+        """
+        Property to declare supported formats.
+
+        Returns:
+            A list or set of supported data formats.
+        """
+        pass