initialize the repo with data downloading, pre-processing and SFT format converting

Author: han032206

configs/config.yaml

@@ -1,60 +1,30 @@
-# 全局配置
+# Global Configuration
 log_dir: "logs"
 
-# 数据下载配置
+# Data Download Configuration
 data_download:
   challenge_id: "L_if1ihd1jmMJq4WUbrYe"
   save_path: "data/raw"
   save_raw_data: true
   username: ""
   password: ""
 
-# 处理模式
-mode: "dom_tree"  # 或 "vision"
+# Processing Mode
+mode: "dom_tree"  # or "vision"
 
-# 数据处理配置
+# Data Processing Configuration
 data_processing:
   save_path: "data/processed/processed_data.json"
   valid_actions: ["click", "type", "hover", "press_enter", "paste", "copy"]
   min_steps: 2
   max_steps: 50
   dom_tree:
     max_sequence_length: 1024
-    include_attributes: ["tag", "text", "href", "src"]
 
   vision:
     image_size: [1024, 1024]
     augmentation: false
-
-# 训练配置
-training:
-  save_path: "data/sft"
-  templates:
-    system_prompt: true
-    include_retrieved_axtree: true
-  dom_tree:
-    model_type: "text-to-text"
-    max_length: 2048
-    
-  vision:
-    planning:
-      model_type: "vision-language"
-      image_encoder: "clip"
-    grounding:
-      model_type: "vision-coordinates"
-      image_size: [1024, 1024]
-
-# 推理配置
-inference:
-  browser_type: "playwright"
-  timeout: 30
-  max_retries: 3
-
-# 评估配置
-evaluation:
-  metrics: ["success_rate", "steps_per_task", "completion_time"]
-  max_steps: 20
-
+  
 # Data paths
 data:
   processed_data_path: "data/processed/processed_data.json"

data_processing/converter/data_converter.py

@@ -6,15 +6,15 @@
 from .utils import download_json, format_node, find_node_by_path, find_node_by_axtid
 
 class SFTConverter:
-    """将处理后的轨迹数据转换为SFT训练格式"""
+    """Convert processed trajectory data to SFT training format"""
 
     def __init__(self, config: Dict):
         self.config = config
         self._setup_logging()
         self._setup_templates()
 
     def _setup_logging(self):
-        """设置日志"""
+        """Initialize logging configuration"""
         log_dir = self.config.get('log_dir', 'logs/converter')
         os.makedirs(log_dir, exist_ok=True)
 
@@ -28,8 +28,8 @@ def _setup_logging(self):
         )
 
     def _setup_templates(self):
-        """设置提示模板"""
-        # 系统提示信息
+        """Setup prompt templates for model input/output"""
+        # System prompt
         self.prompt_system = '''
 # CONTEXT
 
@@ -60,7 +60,7 @@ def _setup_templates(self):
 3. Format actions correctly using the specified structure.
 '''
 
-        # 输入提示模板
+        # Input prompt template
         self.prompt_input_template = '''
 # OBSERVATION
 
@@ -79,22 +79,22 @@ def _setup_templates(self):
 {action_list}
 '''
 
-        # 输出提示模板
+        # Output prompt template
         self.prompt_output_template = '''
 Based on the observation and objective, I will:
 
 {action}
 '''
 
-        # 动作模板
+        # Action template
         self.action_template = '''
 ## Action {i}
 - action_type: {action_type}
 - action_value: {action_value}
 '''
 
     def convert_to_sft_format(self, processed_data: List[Dict]) -> List[Dict]:
-        """转换为SFT训练格式"""
+        """Convert data to SFT training format"""
         sft_data = []
 
         for traj_idx, trajectory in enumerate(processed_data):
@@ -106,33 +106,27 @@ def convert_to_sft_format(self, processed_data: List[Dict]) -> List[Dict]:
                 steps = json.loads(steps_str)
                 logging.info(f"Found {len(steps)} steps in trajectory")
 
-                # 创建轨迹特定的目录
                 traj_dirs = self._create_trajectory_dirs(traj_idx)
 
-                # 处理每个步骤
                 for step_idx, step in enumerate(steps):
                     logging.info(f"Processing step {step_idx}")
 
                     if not self._validate_step(step):
                         continue
 
                     try:
-                        # 处理 axtree 数据
                         formatted_axtree, retrieved_axtree = self._process_axtree(step, traj_idx, step_idx, traj_dirs)
                         if not formatted_axtree or not retrieved_axtree:
                             continue
 
-                        # 构建动作历史
                         action_list = self._build_action_list(steps[:step_idx])
 
-                        # 构建当前动作
                         current_action = {
                             "action_type": step["type"],
                             "action_id": step.get("axtId", ""),
                             "action_value": step.get("value", "")
                         }
 
-                        # 构建训练样本
                         sample = {
                             "prompt_system": self.prompt_system,
                             "prompt_input": self.prompt_input_template.format(
@@ -150,10 +144,8 @@ def convert_to_sft_format(self, processed_data: List[Dict]) -> List[Dict]:
                                 "url": step.get("href", "")
                             }
                         }
-                        
                         sft_data.append(sample)
-                        logging.info(f"Successfully created sample for step {step_idx}")
-                            
+                        
                     except Exception as e:
                         logging.error(f"Error processing step {step_idx}: {str(e)}")
                         continue
@@ -169,16 +161,16 @@ def convert_to_sft_format(self, processed_data: List[Dict]) -> List[Dict]:
         return sft_data
 
     def _validate_step(self, step: Dict) -> bool:
-        """验证步骤数据的有效性"""
-        required_fields = ['type', 'href']  # 移除了 formatted_axtree 的要求
+        """Validate step data completeness"""
+        required_fields = ['type', 'href']
         valid = all(field in step for field in required_fields)
         if not valid:
             missing_fields = [field for field in required_fields if field not in step]
             logging.warning(f"Missing required fields: {missing_fields}")
         return valid
 
     def _build_action_list(self, previous_steps: List[Dict]) -> str:
-        """构建动作历史列表"""
+        """Build history of previous actions"""
         action_list = ""
         for i, step in enumerate(previous_steps):
             action_list += self.action_template.format(
@@ -189,18 +181,17 @@ def _build_action_list(self, previous_steps: List[Dict]) -> str:
         return action_list
 
     def save_sft_data(self, sft_data: List[Dict], output_path: str):
-        """保存SFT训练数据"""
+        """Save SFT training data to JSONL format"""
         os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        # 保存为JSONL格式
         with open(output_path, 'w', encoding='utf-8') as f:
             for sample in sft_data:
                 f.write(json.dumps(sample, ensure_ascii=False) + '\n')
 
         logging.info(f"Saved {len(sft_data)} training samples to {output_path}")
 
     def _create_trajectory_dirs(self, traj_idx: int) -> Dict[str, str]:
-        """创建轨迹相关的目录"""
+        """Create directories for trajectory data"""
         dirs = {
             'raw': os.path.join(self.config.get('raw_axtree_dir', 'data/raw_axtree'), str(traj_idx)),
             'formatted': os.path.join(self.config.get('formatted_axtree_dir', 'data/formatted_axtree'), str(traj_idx)),
@@ -213,41 +204,39 @@ def _create_trajectory_dirs(self, traj_idx: int) -> Dict[str, str]:
         return dirs
 
     def _process_axtree(self, step: Dict, traj_idx: int, step_idx: int, traj_dirs: Dict[str, str]) -> tuple:
-        """处理单个步骤的 axtree 数据"""
+        """Process axtree data for a single step"""
         if not step.get("axTree"):
             logging.warning(f"Step {step_idx} has no axTree data")
             return None, None
 
         try:
-            # 下载和保存原始 axtree
+            # Download and save raw axtree
             raw_path = os.path.join(traj_dirs['raw'], f"{step_idx}.json")
             download_json(step["axTree"], raw_path)
 
-            # 读取原始 axtree
             with open(raw_path, 'r', encoding='utf-8') as f:
                 raw_axtree = json.load(f)
 
-            # 格式化完整 axtree
+            # Format complete axtree
             formatted_nodes = format_node(raw_axtree)
             formatted_axtree = "\n".join(formatted_nodes)
 
-            # 保存格式化后的 axtree
+            # Save formatted axtree
             formatted_path = os.path.join(traj_dirs['formatted'], f"{step_idx}.txt")
             with open(formatted_path, 'w', encoding='utf-8') as f:
                 f.write(formatted_axtree)
 
-            # 查找目标节点
+            # Find target node
             retrieved_node = None
             if "axtId" in step and step["axtId"]:
-                # 优先使用 axtId 查找
                 logging.info(f"Searching node by axtId: {step['axtId']}")
                 retrieved_node = find_node_by_axtid(raw_axtree, step["axtId"])
                 if retrieved_node:
                     logging.info(f"Found node by axtId: {step['axtId']}")
                 else:
                     logging.warning(f"Node not found by axtId: {step['axtId']}, falling back to path search")
 
-            # 如果没有 axtId 或未找到，则使用 path 查找
+            # If no axtId or not found, use path search
             if retrieved_node is None and "path" in step:
                 logging.info(f"Searching node by path: {step['path']}")
                 path = ["html"] + step["path"].split('>')
@@ -261,16 +250,16 @@ def _process_axtree(self, step: Dict, traj_idx: int, step_idx: int, traj_dirs: D
                 logging.warning(f"No node found for step {step_idx}")
                 return formatted_axtree, ""
 
-            # 格式化检索到的节点
+            # Format retrieved node
             retrieved_nodes = format_node(retrieved_node)
             retrieved_axtree = "\n".join(retrieved_nodes)
 
-            # 保存检索到的节点
+            # Save retrieved node
             retrieved_path = os.path.join(traj_dirs['retrieved'], f"{step_idx}.txt")
             with open(retrieved_path, 'w', encoding='utf-8') as f:
                 f.write(retrieved_axtree)
 
-            # 验证找到的节点的 axtId 是否匹配（如果原始步骤中有 axtId）
+            # Verify found node's axtId matches (if original step has axtId)
             if "axtId" in step and step["axtId"]:
                 found_axt_id = retrieved_node.get("attributes", {}).get("data-imean-axt-id")
                 if found_axt_id != step["axtId"]:

data_processing/converter/utils.py

@@ -3,16 +3,14 @@
 
 def download_json(url, output_file='output.json'):
     try:
-        # 发送GET请求获取数据
+        # Send GET request
         response = requests.get(url)
-        response.raise_for_status()  # 检查请求是否成功
+        response.raise_for_status()
 
-        # 将JSON数据保存到文件
+        # Save JSON data to file
         with open(output_file, 'w', encoding='utf-8') as f:
             json.dump(response.json(), f, ensure_ascii=False, indent=2)
 
-        # print(f"Successfully downloaded JSON to {output_file}")
-        
     except requests.exceptions.RequestException as e:
         print(f"Error downloading the file: {e}")
     except json.JSONDecodeError as e:
@@ -21,21 +19,23 @@ def download_json(url, output_file='output.json'):
 
 def find_node_by_axtid(node, axt_id):
     """
-    递归遍历 axtree，寻找指定 axt_id 的节点。
+    Recursively traverse axtree to find node with specified axt_id.
     
-    :param node: 当前节点
-    :param axt_id: 要查找的 axt_id
-    :return: 如果找到匹配的节点，返回节点对象；否则返回 None
+    Args:
+        node: Current node
+        axt_id: Target axt_id to find
+    Returns:
+        Matching node object if found, None otherwise
     """
     if node is None:
         return None
 
-    # 检查当前节点的 axt_id
+    # Check current node's axt_id
     current_axt_id = node.get("attributes", {}).get("data-imean-axt-id")
     if current_axt_id == axt_id:
         return node
 
-    # 递归检查子节点
+    # Check child nodes recursively
     for child in node.get("children", []):
         result = find_node_by_axtid(child, axt_id)
         if result:
@@ -45,46 +45,41 @@ def find_node_by_axtid(node, axt_id):
 
 def find_node_by_path(node, path, current_level=0):
     """
-    递归遍历 axtree，寻找路径为 path 的节点。
-    作为备选方案，当 axtId 不存在或未找到时使用。
+    Recursively traverse axtree to find node at specified path.
+    Used as fallback when axtId is not available or not found.
     """
     if node is None:
         return None
 
-    # 获取当前节点的标签
     html_tag = node.get("attributes", {}).get("html_tag", "")
 
-    # 检查当前节点是否匹配路径的当前部分
     if html_tag != path[current_level]:
         return None
 
-    # 如果已经匹配到路径的最后一级，返回当前节点
     if current_level == len(path) - 1:
         return node
 
-    # 遍历子节点，递归查找下一层级
     for child in node.get("children", []):
         result = find_node_by_path(child, path, current_level + 1)
         if result:
             return result
 
-    # 如果没有找到，返回 None
     return None
 
 def format_node(node, level=0):
+    """Format node and its children into a readable tree structure"""
     result = []
-    indent = "  " * level  # 2 spaces per level
+    indent = "  " * level
 
     if node is None:
         return result
 
-    # Get attributes for current node
+    # Get node attributes
     axt_id = node.get("attributes", {}).get("data-imean-axt-id")
     role = node.get("role")
     name = node.get("name")
     value = node.get("value")
 
-    # Add formatted string if node has all required attributes
     if axt_id and role:
         formatted = indent + f"[{axt_id}] {role}"
         if name:
@@ -93,7 +88,7 @@ def format_node(node, level=0):
             formatted += f" '{value}'"
         result.append(formatted)
 
-    # Recursively process children
+    # Process children recursively
     children = node.get("children", [])
     for child in children:
         result.extend(format_node(child, level + 1))