Fix: Add error handling for tokenizers without chat templates

Author: philipithomas

tests/generate_tests.py

@@ -348,31 +348,41 @@ def generate_tokenizer_tests():
     for tokenizer_id in TOKENIZERS_WITH_CHAT_TEMPLATES:
         print(f'Generating chat templates for {tokenizer_id}')
 
-        # In local mode, use safer settings
-        use_fast = not LOCAL_MODE or 'llama' not in tokenizer_id
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            tokenizer_id,
-            use_fast=use_fast,
-            trust_remote_code=True,
-        )
-        tokenizer_results = []
-        for key in TOKENIZERS_WITH_CHAT_TEMPLATES[tokenizer_id]:
-            messages = CHAT_MESSAGES_EXAMPLES[key]
-
-            for add_generation_prompt, tokenize in product([True, False], [True, False]):
-                tokenizer_results.append(dict(
-                    messages=messages,
-                    add_generation_prompt=add_generation_prompt,
-                    tokenize=tokenize,
-                    target=tokenizer.apply_chat_template(
-                        messages,
-                        add_generation_prompt=add_generation_prompt,
-                        tokenize=tokenize,
-                    ),
-                ))
+        try:
+            # In local mode, use safer settings
+            use_fast = not LOCAL_MODE or 'llama' not in tokenizer_id
+
+            tokenizer = AutoTokenizer.from_pretrained(
+                tokenizer_id,
+                use_fast=use_fast,
+                trust_remote_code=True,
+            )
+            tokenizer_results = []
+            for key in TOKENIZERS_WITH_CHAT_TEMPLATES[tokenizer_id]:
+                messages = CHAT_MESSAGES_EXAMPLES[key]
+
+                for add_generation_prompt, tokenize in product([True, False], [True, False]):
+                    try:
+                        result = tokenizer.apply_chat_template(
+                            messages,
+                            add_generation_prompt=add_generation_prompt,
+                            tokenize=tokenize,
+                        )
+                        tokenizer_results.append(dict(
+                            messages=messages,
+                            add_generation_prompt=add_generation_prompt,
+                            tokenize=tokenize,
+                            target=result,
+                        ))
+                    except ValueError as e:
+                        print(f"  - Skipping template for {tokenizer_id} with {key}: {str(e)}")
+                        continue
 
-        template_results[tokenizer_id] = tokenizer_results
+            if tokenizer_results:
+                template_results[tokenizer_id] = tokenizer_results
+        except Exception as e:
+            print(f"  - Error processing tokenizer {tokenizer_id}: {str(e)}")
+            continue
 
     return dict(
         tokenization=tokenization_results,
@@ -455,3 +465,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+