1#!/usr/bin/env python3
2"""Validate SFT (Supervised Fine-Tuning) JSONL files for Azure AI Foundry.
3 
4Adapted from foundry-ft agent with additional checks from our platform gotchas:
5- Token length warnings (4096 limit varies by model)
6- System prompt consistency check
7"""
8import json
9import sys
10 
11 
12try:
13    sys.stdout.reconfigure(encoding="utf-8")
14    sys.stderr.reconfigure(encoding="utf-8")
15except (AttributeError, OSError):
16    pass  # Stream not reconfigurable (older Python or non-tty); default encoding is fine
17VALID_ROLES = {"system", "user", "assistant", "tool"}
18 
19 
20def estimate_tokens(text: str) -> int:
21    """Rough token estimate: ~4 chars per token for English text."""
22    return max(1, len(text) // 4)
23 
24 
25def validate_sft(filepath: str) -> None:
26    errors = []
27    warnings = []
28    total = 0
29    token_counts = []
30    system_prompts = set()
31 
32    with open(filepath, "r", encoding="utf-8") as f:
33        for line_num, line in enumerate(f, 1):
34            line = line.strip()
35            if not line:
36                continue
37            total += 1
38 
39            try:
40                record = json.loads(line)
41            except json.JSONDecodeError as e:
42                errors.append(f"Line {line_num}: Invalid JSON — {e}")
43                continue
44 
45            if "messages" not in record:
46                errors.append(f"Line {line_num}: Missing 'messages' field")
47                continue
48 
49            messages = record["messages"]
50            if not isinstance(messages, list) or len(messages) == 0:
51                errors.append(f"Line {line_num}: 'messages' must be a non-empty array")
52                continue
53 
54            roles_found = set()
55            total_text = ""
56            for i, msg in enumerate(messages):
57                if "role" not in msg:
58                    errors.append(f"Line {line_num}, message {i}: Missing 'role'")
59                elif msg["role"] not in VALID_ROLES:
60                    errors.append(f"Line {line_num}, message {i}: Invalid role '{msg['role']}' (expected: {VALID_ROLES})")
61                else:
62                    roles_found.add(msg["role"])
63 
64                if "content" not in msg and "tool_calls" not in msg:
65                    errors.append(f"Line {line_num}, message {i}: Missing 'content' (and no 'tool_calls')")
66                elif "content" in msg and msg["content"] is not None:
67                    content = str(msg["content"])
68                    if not content.strip():
69                        warnings.append(f"Line {line_num}, message {i}: Empty content string")
70                    total_text += content
71 
72                    if msg.get("role") == "system":
73                        system_prompts.add(content.strip()[:100])
74 
75            if "user" not in roles_found:
76                errors.append(f"Line {line_num}: No 'user' message found")
77            if "assistant" not in roles_found:
78                errors.append(f"Line {line_num}: No 'assistant' message found")
79 
80            tokens = estimate_tokens(total_text)
81            token_counts.append(tokens)
82            if tokens > 4096:
83                warnings.append(f"Line {line_num}: ~{tokens} tokens (exceeds 4096 limit for most models)")
84 
85    # Report
86    print(f"\n{'='*60}")
87    print(f"SFT Validation Report: {filepath}")
88    print(f"{'='*60}")
89    print(f"Total records: {total}")
90    print(f"Errors: {len(errors)}")
91    print(f"Warnings: {len(warnings)}")
92 
93    if token_counts:
94        avg_tok = sum(token_counts) / len(token_counts)
95        print(f"\nToken stats (approx):")
96        print(f"  Avg: {avg_tok:.0f}  Min: {min(token_counts)}  Max: {max(token_counts)}")
97        print(f"  Total: {sum(token_counts):,}")
98 
99    if len(system_prompts) > 1:
100        warnings.append(f"Found {len(system_prompts)} different system prompts — ensure this is intentional")
101    if system_prompts:
102        print(f"\nSystem prompts: {len(system_prompts)} unique")
103 
104    if errors:
105        print(f"\n❌ ERRORS (must fix):")
106        for e in errors[:20]:
107            print(f"  • {e}")
108        if len(errors) > 20:
109            print(f"  ... and {len(errors) - 20} more errors")
110 
111    if warnings:
112        print(f"\n⚠️  WARNINGS:")
113        for w in warnings[:10]:
114            print(f"  • {w}")
115 
116    if not errors:
117        print(f"\n✅ Data is valid for SFT fine-tuning!")
118    else:
119        print(f"\n❌ Fix {len(errors)} error(s) before submitting.")
120        sys.exit(1)
121 
122 
123if __name__ == "__main__":
124    if len(sys.argv) != 2:
125        print("Usage: python validate_sft.py <path-to-jsonl>")
126        sys.exit(1)
127    validate_sft(sys.argv[1])
128