1import { tool } from 'ai';
2import { z } from 'zod';
3import { openai } from '@ai-sdk/openai';
4import { generateText } from 'ai';
5import { config } from '../../config/index.js';
6 
7export const PairwiseCompareInputSchema = z.object({
8  responseA: z.string().describe('First response to compare'),
9  responseB: z.string().describe('Second response to compare'),
10  prompt: z.string().describe('The original prompt both responses address'),
11  context: z.string().optional().describe('Additional context'),
12  criteria: z.array(z.string()).min(1).describe('Comparison criteria'),
13  allowTie: z.boolean().optional().default(true).describe('Allow tie verdict'),
14  swapPositions: z.boolean().optional().default(true).describe('Swap positions to reduce bias')
15});
16 
17export type PairwiseCompareInput = z.infer<typeof PairwiseCompareInputSchema>;
18 
19export const PairwiseCompareOutputSchema = z.object({
20  success: z.boolean(),
21  winner: z.enum(['A', 'B', 'TIE']),
22  confidence: z.number().min(0).max(1),
23  comparison: z.array(z.object({
24    criterion: z.string(),
25    winner: z.enum(['A', 'B', 'TIE']),
26    aAssessment: z.string(),
27    bAssessment: z.string(),
28    reasoning: z.string()
29  })),
30  analysis: z.object({
31    responseA: z.object({
32      strengths: z.array(z.string()),
33      weaknesses: z.array(z.string())
34    }),
35    responseB: z.object({
36      strengths: z.array(z.string()),
37      weaknesses: z.array(z.string())
38    })
39  }),
40  differentiators: z.array(z.string()),
41  positionConsistency: z.object({
42    consistent: z.boolean(),
43    firstPassWinner: z.enum(['A', 'B', 'TIE']).optional(),
44    secondPassWinner: z.enum(['A', 'B', 'TIE']).optional()
45  }).optional(),
46  metadata: z.object({
47    evaluationTimeMs: z.number(),
48    model: z.string(),
49    positionsSwapped: z.boolean()
50  })
51});
52 
53export type PairwiseCompareOutput = z.infer<typeof PairwiseCompareOutputSchema>;
54 
55async function evaluatePair(
56  first: string,
57  second: string,
58  prompt: string,
59  criteria: string[],
60  context?: string,
61  allowTie: boolean = true
62): Promise<{ winner: 'A' | 'B' | 'TIE'; confidence: number; comparison: PairwiseCompareOutput['comparison']; analysis: PairwiseCompareOutput['analysis'] }> {
63  const systemPrompt = `You are an expert evaluator comparing two AI responses.
64 
65IMPORTANT:
66- Do NOT prefer responses because they are longer
67- Do NOT prefer responses based on position (first vs second)
68- Focus only on quality according to the criteria
69- ${allowTie ? 'Ties are acceptable when responses are genuinely equivalent' : 'You must choose a winner'}`;
70 
71  const userPrompt = `## Original Prompt
72${prompt}
73 
74${context ? `## Context\n${context}\n` : ''}
75## Response A
76${first}
77 
78## Response B
79${second}
80 
81## Criteria to Compare
82${criteria.map((c, i) => `${i + 1}. ${c}`).join('\n')}
83 
84First analyze each response independently, then compare them.
85Respond with valid JSON:
86{
87  "analysis": {
88    "responseA": { "strengths": [...], "weaknesses": [...] },
89    "responseB": { "strengths": [...], "weaknesses": [...] }
90  },
91  "comparison": [
92    {
93      "criterion": "criterion name",
94      "winner": "A" | "B" | "TIE",
95      "aAssessment": "brief assessment of A",
96      "bAssessment": "brief assessment of B",
97      "reasoning": "why this winner"
98    }
99  ],
100  "result": {
101    "winner": "A" | "B" | "TIE",
102    "confidence": 0.0-1.0,
103    "reasoning": "overall reasoning"
104  }
105}`;
106 
107  const result = await generateText({
108    model: openai(config.openai.model),
109    system: systemPrompt,
110    prompt: userPrompt,
111    temperature: 0.3
112  });
113 
114  const parsed = JSON.parse(result.text);
115  
116  return {
117    winner: parsed.result.winner,
118    confidence: parsed.result.confidence,
119    comparison: parsed.comparison,
120    analysis: parsed.analysis
121  };
122}
123 
124export async function executePairwiseCompare(input: PairwiseCompareInput): Promise<PairwiseCompareOutput> {
125  const startTime = Date.now();
126 
127  try {
128    if (input.swapPositions) {
129      // First pass: A first, B second
130      const pass1 = await evaluatePair(
131        input.responseA,
132        input.responseB,
133        input.prompt,
134        input.criteria,
135        input.context,
136        input.allowTie
137      );
138 
139      // Second pass: B first, A second
140      const pass2 = await evaluatePair(
141        input.responseB,
142        input.responseA,
143        input.prompt,
144        input.criteria,
145        input.context,
146        input.allowTie
147      );
148 
149      // Map pass2 result back
150      const pass2WinnerMapped = pass2.winner === 'A' ? 'B' : pass2.winner === 'B' ? 'A' : 'TIE';
151      const consistent = pass1.winner === pass2WinnerMapped;
152 
153      // Determine final winner
154      let finalWinner: 'A' | 'B' | 'TIE';
155      let finalConfidence: number;
156 
157      if (consistent) {
158        finalWinner = pass1.winner;
159        finalConfidence = (pass1.confidence + pass2.confidence) / 2;
160      } else {
161        // Inconsistent - return tie with lower confidence
162        finalWinner = 'TIE';
163        finalConfidence = 0.5;
164      }
165 
166      // Merge comparisons
167      const mergedComparison = pass1.comparison.map((c, i) => {
168        const c2 = pass2.comparison[i];
169        const c2WinnerMapped = c2.winner === 'A' ? 'B' : c2.winner === 'B' ? 'A' : 'TIE';
170        return {
171          ...c,
172          winner: c.winner === c2WinnerMapped ? c.winner : 'TIE' as const
173        };
174      });
175 
176      // Find differentiators
177      const differentiators = mergedComparison
178        .filter(c => c.winner !== 'TIE')
179        .map(c => `${c.criterion}: ${c.winner === 'A' ? 'Response A' : 'Response B'} wins - ${c.reasoning}`);
180 
181      return {
182        success: true,
183        winner: finalWinner,
184        confidence: Math.round(finalConfidence * 100) / 100,
185        comparison: mergedComparison,
186        analysis: pass1.analysis,
187        differentiators,
188        positionConsistency: {
189          consistent,
190          firstPassWinner: pass1.winner,
191          secondPassWinner: pass2WinnerMapped
192        },
193        metadata: {
194          evaluationTimeMs: Date.now() - startTime,
195          model: config.openai.model,
196          positionsSwapped: true
197        }
198      };
199    } else {
200      // Single pass without swap
201      const result = await evaluatePair(
202        input.responseA,
203        input.responseB,
204        input.prompt,
205        input.criteria,
206        input.context,
207        input.allowTie
208      );
209 
210      const differentiators = result.comparison
211        .filter(c => c.winner !== 'TIE')
212        .map(c => `${c.criterion}: ${c.winner === 'A' ? 'Response A' : 'Response B'} wins - ${c.reasoning}`);
213 
214      return {
215        success: true,
216        winner: result.winner,
217        confidence: result.confidence,
218        comparison: result.comparison,
219        analysis: result.analysis,
220        differentiators,
221        metadata: {
222          evaluationTimeMs: Date.now() - startTime,
223          model: config.openai.model,
224          positionsSwapped: false
225        }
226      };
227    }
228  } catch (error) {
229    return {
230      success: false,
231      winner: 'TIE',
232      confidence: 0,
233      comparison: [],
234      analysis: {
235        responseA: { strengths: [], weaknesses: [] },
236        responseB: { strengths: [], weaknesses: [] }
237      },
238      differentiators: [],
239      metadata: {
240        evaluationTimeMs: Date.now() - startTime,
241        model: config.openai.model,
242        positionsSwapped: input.swapPositions
243      }
244    };
245  }
246}
247 
248export const pairwiseCompareTool = tool({
249  description: `Compare two responses and select the better one.
250Use for subjective evaluations like tone, persuasiveness, style.
251More reliable than direct scoring for preferences.`,
252  parameters: PairwiseCompareInputSchema,
253  execute: executePairwiseCompare
254});
255 
256