refactoring of fetch node

Author: VinciGit00

scrapegraphai/nodes/fetch_html_node.py

@@ -2,8 +2,8 @@
 Module for fetching the HTML node
 """
 
-from langchain_community.document_loaders import AsyncHtmlLoader
 from .base_node import BaseNode
+from ..utils.getter import _get_function
 
 
 class FetchHTMLNode(BaseNode):
@@ -60,8 +60,9 @@ def execute(self, state: dict) -> dict:
         except KeyError as e:
             print(f"Error: {e} not found in state.")
             raise
-        loader = AsyncHtmlLoader(url)
-        document = loader.load()
+
+        document = _get_function(url)
+
         state["keys"]["document"] = document
 
         return state

scrapegraphai/nodes/image_to_text_node.py

@@ -1,11 +1,13 @@
 """ 
 Module for creating the basic node
 """
-from abc import ABC, abstractmethod
-from openai import OpenAI
+from langchain_openai import ChatOpenAI
+from .base_node import BaseNode
 
+from langchain_core.messages import HumanMessage, SystemMessage
 
-class ImageToTextNode(ABC):
+
+class ImageToTextNode(BaseNode):
     """
     An abstract base class for nodes in a graph-based workflow. Each node is 
     intended to perform a specific action when executed as part of the graph's 
@@ -53,7 +55,6 @@ def __init__(self, llm, node_name: str = "ParseImageToText"):
         super().__init__(node_name, "node")
         self.llm = llm
 
-    @abstractmethod
     def execute(self, state: dict, url: str) -> str:
         """
         Execute the node's logic and return the updated state.
@@ -63,28 +64,26 @@ def execute(self, state: dict, url: str) -> str:
         :return: The updated state after executing this node.
         """
         # Da fixare
-        client = OpenAI(api_key=self.llm.openai_api_key)
 
         if not self.llm.model_name == "gpt-4-vision-preview":
             raise ValueError("Model is not gpt-4-vision-preview")
 
-        response = client.chat.completions.create(
-            model=self.llm.model_name,
-            messages=[
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "text", "text": "What’s in this image?"},
+        chat = ChatOpenAI(model=self.llm.model_name, max_tokens=256)
+        result = chat.invoke(
+            [
+                HumanMessage(
+                    content=[
+                        {"type": "text", "text": "What is this image showing"},
                         {
                             "type": "image_url",
                             "image_url": {
                                 "url": url,
+                                "detail": "auto",
                             },
                         },
-                    ],
-                }
-            ],
-            max_tokens=300,
+                    ]
+                )
+            ]
         )
 
-        return response.choices[0]
+        return result

scrapegraphai/utils/getter.py

@@ -14,5 +14,6 @@ def _get_function(link: str) -> str:
     Returns:
         str: The content of the response as a string.
     """
+
     loader = AsyncHtmlLoader(link)
-    return str(loader.load())
+    return loader.load()