Merge pull request #2 from iMeanAI/dev

update a demo base agent

Author: han032206

.gitignore

@@ -46,6 +46,8 @@ logs/
 log/
 tmp/
 temp/
+token_results/
+temp_ref/
 
 # Data
 data/

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

README.md

@@ -13,7 +13,7 @@
 
 # open-operator
 
-This project aims to provide the open-source community with an easy-to-use system for building, self-hosting, and evaluating web agent models. Our goal is to offer an alternative to the $200/month ChatGPT Pro and cloud-based, uncontrolled execution environments.
+This project aims to provide the open-source community with an easy-to-use system for building, self-hosting, and evaluating web agent computer-use models. Our goal is to offer an alternative to the $200/month ChatGPT Pro and cloud-based, uncontrolled execution environments.
 
 With open-operator, you can:
 - Annotate your web trajectory data.
@@ -28,6 +28,30 @@ We believe in empowering developers to have complete control over their web agen
 ![Roadmap](src/roadmap.png)
 Briefly describe the roadmap of the project. Green part will be included in this repo.
 
+## Run your Base Agent Using Open-Operator
+### Prepare the environment
+```bash
+conda create -n open-operator python=3.11
+pip install -r requirements.txt
+```
+
+For the browser environment, you can use [browserbase](https://www.browserbase.com/) to setup the following environment variables.
+
+```bash
+export BROWSERBASE_API_KEY=your_api_key
+```
+
+
+### Initialize the base agent
+```bash
+python inference/app.py
+```
+You can select the base model you want to use in the dropdown menu.(From Anthropic, Google, OpenAI, etc.)
+
+![Open-Operator](src/ui.png)
+
+Then start your first experience with Open-Operator!
+
 ## Data Annotation and Downloading
 Follow the step wise instruction below:
 1. Download the latest iMean builder extension here: [iMean Builder](https://drive.google.com/file/d/1BpLOQ9M41rdc6VYY-1Aes1lhzo5-LdiH/view?usp=sharing)

configs/config.yaml

@@ -6,8 +6,8 @@ data_download:
   challenge_id: "L_if1ihd1jmMJq4WUbrYe"
   save_path: "data/raw"
   save_raw_data: true
-  username: ""
-  password: ""
+  username: "han032206@gmail.com"
+  password: "qazwsxpl,okm145"
 
 # Processing Mode
 mode: "dom_tree"  # or "vision"

inference/agent/Environment/__init__.py

@@ -0,0 +1 @@
+from .html_env import *

inference/agent/Environment/browser_env/__init__.py

@@ -0,0 +1,38 @@
+from enum import Enum
+from playwright.async_api import async_playwright
+import os
+
+class BrowserType(Enum):
+    LOCAL = "local"
+    BROWSERBASE = "browserbase"
+
+class BrowserEnvironment:
+    @staticmethod
+    async def create_browser_instance(browser_type: str, headless: bool, slow_mo: int, viewport_size: dict, locale: str):
+        if browser_type == BrowserType.LOCAL.value:
+            return await LocalBrowserEnvironment.create(headless, slow_mo, viewport_size, locale)
+        elif browser_type == BrowserType.BROWSERBASE.value:
+            return await BrowserbaseBrowserEnvironment.create(headless, slow_mo, viewport_size, locale)
+        else:
+            raise ValueError(f"Unsupported browser type: {browser_type}")
+
+class LocalBrowserEnvironment:
+    @staticmethod
+    async def create(headless: bool, slow_mo: int, viewport_size: dict, locale: str):
+        playwright = await async_playwright().start()
+        browser = await playwright.chromium.launch(headless=headless, slow_mo=slow_mo)
+        context = await browser.new_context(viewport=viewport_size, locale=locale)
+        return browser, context, playwright, None
+
+class BrowserbaseBrowserEnvironment:
+    @staticmethod
+    async def create(headless: bool, slow_mo: int, viewport_size: dict, locale: str):
+        playwright = await async_playwright().start()
+        browserbase_api_key = os.environ.get('BROWSERBASE_API_KEY')
+        if not browserbase_api_key:
+            raise ValueError("BROWSERBASE_API_KEY not found in environment variables")
+        
+        browser_cdp_url = f"wss://connect.browserbase.com?apiKey={browserbase_api_key}"
+        browser = await playwright.chromium.connect_over_cdp(browser_cdp_url)
+        context = browser.contexts[0]
+        return browser, context, playwright, None

inference/agent/Environment/html_env/__init__.py

@@ -0,0 +1,5 @@
+from .utils import *
+from .build_tree import *
+from .active_elements import *
+from .actions import *
+from .async_env import *

inference/agent/Environment/html_env/actions.py

@@ -0,0 +1,216 @@
+from beartype import beartype
+from typing import TypedDict
+from enum import IntEnum
+
+
+class Action(TypedDict):
+    action_type: int
+    element_id: int
+    element_name: str
+    url: str
+    fill_text: str
+
+
+class ActionTypes(IntEnum):
+    NONE = 0
+    CLICK = 1
+    GOTO = 2
+    GOOGLE_SEARCH = 3
+    FILL_FORM = 4
+    SWITCH_TAB = 5
+    GO_BACK = 6
+    FILL_SEARCH = 7
+    SELECT_OPTION = 8
+    HOVER = 9
+    SCROLL_DOWN = 10
+    SCROLL_UP = 11
+    CACHE_DATA = 12
+    GET_FINAL_ANSWER = 13
+
+@beartype
+def create_cache_data_action(elementid: int,fill_text: str) -> Action:
+    return {
+        "action_type": ActionTypes.CACHE_DATA,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": fill_text,
+        "element_name": ""
+    }
+
+
+@beartype
+def create_get_final_answer(elementid: int,fill_text: str) -> Action:
+    return {
+        "action_type": ActionTypes.GET_FINAL_ANSWER,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": fill_text,
+        "element_name": ""
+    }
+
+
+@beartype
+def create_click_action(elementid: int) -> Action:
+    return {
+        "action_type": ActionTypes.CLICK,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": "",
+        "element_name": ""
+    }
+
+
+@beartype
+def create_goto_action(elementid: int, url: str) -> Action:
+    return {
+        "action_type": ActionTypes.GOTO,
+        "element_id": elementid,
+        "url": url,
+        "fill_text": "",
+        "element_name": ""
+    }
+
+
+@beartype
+def create_none_action(elementid: int) -> Action:
+    return {
+        "action_type": ActionTypes.NONE,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": "",
+        "element_name": ""
+    }
+
+
+@beartype
+def create_fill_action(elementid: int, fill_text: str) -> Action:
+    return {
+        "action_type": ActionTypes.FILL_FORM,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": fill_text,
+        "element_name": ""
+    }
+
+
+@beartype
+def create_fill_search_action(elementid: int, fill_text: str) -> Action:
+    return {
+        "action_type": ActionTypes.FILL_SEARCH,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": fill_text,
+        "element_name": ""
+    }
+
+
+@beartype
+def create_search_action(elementid: int, text: str) -> Action:
+    return {
+        "action_type": ActionTypes.GOOGLE_SEARCH,
+        "element_id": elementid,
+        "url": "https://www.google.com",
+        "fill_text": text,
+        "element_name": ""
+    }
+
+
+@beartype
+def create_go_back_action(elementid: int) -> Action:
+    return {
+        "action_type": ActionTypes.GO_BACK,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": "",
+        "element_name": ""
+    }
+
+
+@beartype
+def create_select_option_action(elementid: int, target_value: str) -> Action:
+    return {
+        "action_type": ActionTypes.SELECT_OPTION,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": target_value,
+        "element_name": ""
+    }
+
+@beartype
+def create_hover_action(elementid: int) -> Action:
+    return {
+        "action_type": ActionTypes.HOVER,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": "",
+        "element_name": ""
+    }
+
+@beartype
+def create_scroll_down_action(elementid: int) -> Action:
+    return {
+        "action_type": ActionTypes.SCROLL_DOWN,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": "",
+        "element_name": ""
+    }
+
+@beartype
+def create_scroll_up_action(elementid: int) -> Action:
+    return {
+        "action_type": ActionTypes.SCROLL_UP,
+        "element_id": elementid,
+        "url": "",
+        "fill_text": "",
+        "element_name": ""
+    }
+
+@beartype
+def create_action(elementid: int, action_type: str, action_input: str) -> Action:
+    if action_type == "click":
+        return create_click_action(elementid=elementid)
+    elif action_type == "fill_form":
+        return create_fill_action(elementid=elementid, fill_text=action_input)
+    elif action_type == "fill_search":
+        return create_fill_search_action(elementid=elementid, fill_text=action_input)
+    elif action_type == "goto":
+        return create_goto_action(elementid=elementid, url=action_input)
+    elif action_type == "google_search":
+        return create_search_action(elementid=elementid, text=action_input)
+    elif action_type == "go_back":
+        return create_go_back_action(elementid=elementid)
+    elif action_type == "select_option":
+        return create_select_option_action(elementid=elementid, target_value=action_input)
+    elif action_type == "hover":
+        return create_hover_action(elementid=elementid)
+    elif action_type == "scroll_down":
+        return create_scroll_down_action(elementid=elementid)
+    elif action_type == "scroll_up":
+        return create_scroll_up_action(elementid=elementid)
+    elif action_type == "cache_storage":
+        return create_cache_data_action(elementid=elementid,fill_text=action_input)
+    elif action_type == "get_final_answer":
+        return create_get_final_answer(elementid=elementid,fill_text=action_input)
+    else:
+        return create_none_action(elementid=elementid)
+
+
+__all__ = [
+    "Action",
+    "ActionTypes",
+    "create_click_action",
+    "create_fill_action",
+    "create_none_action",
+    "create_goto_action",
+    "create_search_action",
+    "create_go_back_action",
+    "create_fill_search_action",
+    "create_select_option_action",
+    "create_hover_action",
+    "create_scroll_down_action",
+    "create_scroll_up_action",
+    "create_cache_data_action",
+    "create_get_final_answer",
+    "create_action"
+]