1/**
2 * Full Evaluation Workflow Example
3 * 
4 * Demonstrates a complete evaluation workflow:
5 * 1. Generate rubrics for criteria
6 * 2. Score a response using generated rubrics
7 * 3. Compare with an alternative response
8 * 
9 * Run: npx tsx examples/full-evaluation-workflow.ts
10 */
11 
12import 'dotenv/config';
13import { EvaluatorAgent } from '../src/agents/evaluator.js';
14import { validateConfig } from '../src/config/index.js';
15 
16async function main() {
17  validateConfig();
18 
19  const agent = new EvaluatorAgent();
20  const startTime = Date.now();
21 
22  console.log('=== Full Evaluation Workflow ===\n');
23 
24  const prompt = 'Explain how vaccines work to prevent disease';
25 
26  const response = `
27    Vaccines work by training your immune system to recognize and fight specific pathogens
28    without causing the disease itself.
29    
30    Here's the process:
31    
32    1. **Introduction**: The vaccine introduces a weakened, killed, or partial version of
33       the pathogen (or instructions to make a piece of it, like mRNA vaccines).
34    
35    2. **Immune Response**: Your immune system detects these foreign substances (antigens)
36       and mounts a response. This includes producing antibodies and training T-cells.
37    
38    3. **Memory Formation**: Some immune cells become "memory cells" that remember
39       how to fight this specific pathogen.
40    
41    4. **Future Protection**: If you're exposed to the real pathogen later, your immune
42       system recognizes it immediately and can fight it off before you get sick.
43    
44    This is why vaccines are so effective - they give your immune system a "practice run"
45    without the risks of actual infection.
46  `;
47 
48  // Step 1: Generate rubrics
49  console.log('Step 1: Generating rubrics...\n');
50 
51  const criteria = [
52    { name: 'Scientific Accuracy', description: 'Correctness of biological/medical information' },
53    { name: 'Completeness', description: 'Covers the key steps and concepts' },
54    { name: 'Accessibility', description: 'Understandable by general audience' }
55  ];
56 
57  const rubrics = await Promise.all(
58    criteria.map(c => agent.generateRubric({
59      criterionName: c.name,
60      criterionDescription: c.description,
61      scale: '1-5',
62      domain: 'health education',
63      includeExamples: false,
64      strictness: 'balanced'
65    }))
66  );
67 
68  console.log('Generated rubrics for:');
69  rubrics.forEach(r => {
70    if (r.success) {
71      console.log(`  - ${r.criterion.name} (${r.levels.length} levels)`);
72    }
73  });
74 
75  // Step 2: Score the response
76  console.log('\nStep 2: Scoring the response...\n');
77 
78  const scoreResult = await agent.score({
79    response,
80    prompt,
81    criteria: criteria.map((c, i) => ({
82      name: c.name,
83      description: c.description,
84      weight: i === 0 ? 0.4 : 0.3 // Weight accuracy higher
85    })),
86    rubric: {
87      scale: '1-5',
88      levelDescriptions: rubrics[0].success 
89        ? Object.fromEntries(rubrics[0].levels.map(l => [String(l.score), l.label]))
90        : undefined
91    }
92  });
93 
94  if (scoreResult.success) {
95    console.log('Scores:');
96    scoreResult.scores.forEach(s => {
97      console.log(`  ${s.criterion}: ${s.score}/${s.maxScore}`);
98    });
99    console.log(`\nOverall: ${scoreResult.overallScore} | Weighted: ${scoreResult.weightedScore}`);
100  }
101 
102  // Step 3: Compare with an alternative
103  console.log('\nStep 3: Comparing with alternative response...\n');
104 
105  const alternativeResponse = `
106    Vaccines prevent disease by helping your body build immunity. When you get
107    vaccinated, your body learns to fight the germ. Then if you're exposed to
108    the real disease, your body already knows how to protect itself.
109  `;
110 
111  const comparisonResult = await agent.compare({
112    responseA: response,
113    responseB: alternativeResponse,
114    prompt,
115    criteria: ['accuracy', 'depth', 'clarity'],
116    swapPositions: true
117  });
118 
119  if (comparisonResult.success) {
120    console.log(`Comparison Result: Response ${comparisonResult.winner} is better`);
121    console.log(`Confidence: ${(comparisonResult.confidence * 100).toFixed(0)}%`);
122    console.log('\nKey differences:');
123    comparisonResult.differentiators.slice(0, 3).forEach(d => console.log(`  - ${d}`));
124  }
125 
126  // Summary
127  const totalTime = Date.now() - startTime;
128  console.log('\n=== Workflow Complete ===');
129  console.log(`Total time: ${totalTime}ms`);
130  console.log(`Rubrics generated: ${rubrics.filter(r => r.success).length}`);
131  console.log(`Final score: ${scoreResult.success ? scoreResult.overallScore : 'N/A'}`);
132  console.log(`Better response: ${comparisonResult.success ? comparisonResult.winner : 'N/A'}`);
133}
134 
135main().catch(console.error);
136 
137