1import { tool } from 'ai';
2import { z } from 'zod';
3import { openai } from '@ai-sdk/openai';
4import { generateText } from 'ai';
5import { config } from '../../config/index.js';
6 
7export const GenerateRubricInputSchema = z.object({
8  criterionName: z.string().describe('Name of the criterion'),
9  criterionDescription: z.string().describe('What this criterion measures'),
10  scale: z.enum(['1-3', '1-5', '1-10']).optional().default('1-5'),
11  domain: z.string().optional().describe('Domain context'),
12  includeExamples: z.boolean().optional().default(true),
13  strictness: z.enum(['lenient', 'balanced', 'strict']).optional().default('balanced')
14});
15 
16export type GenerateRubricInput = z.infer<typeof GenerateRubricInputSchema>;
17 
18export const GenerateRubricOutputSchema = z.object({
19  success: z.boolean(),
20  criterion: z.object({
21    name: z.string(),
22    description: z.string()
23  }),
24  scale: z.object({
25    min: z.number(),
26    max: z.number(),
27    type: z.string()
28  }),
29  levels: z.array(z.object({
30    score: z.number(),
31    label: z.string(),
32    description: z.string(),
33    characteristics: z.array(z.string()),
34    example: z.string().optional()
35  })),
36  scoringGuidelines: z.array(z.string()),
37  edgeCases: z.array(z.object({
38    situation: z.string(),
39    guidance: z.string()
40  })),
41  metadata: z.object({
42    domain: z.string().nullable(),
43    strictness: z.string(),
44    generationTimeMs: z.number()
45  })
46});
47 
48export type GenerateRubricOutput = z.infer<typeof GenerateRubricOutputSchema>;
49 
50export async function executeGenerateRubric(input: GenerateRubricInput): Promise<GenerateRubricOutput> {
51  const startTime = Date.now();
52  const [minScore, maxScore] = input.scale.split('-').map(Number);
53 
54  const systemPrompt = `You are an expert in creating evaluation rubrics.
55Create clear, actionable rubrics with distinct boundaries between levels.
56Strictness: ${input.strictness}
57- lenient: Lower bar for passing scores
58- balanced: Fair, typical expectations
59- strict: High standards, critical evaluation`;
60 
61  const userPrompt = `Create a scoring rubric for:
62 
63**Criterion**: ${input.criterionName}
64**Description**: ${input.criterionDescription}
65**Scale**: ${input.scale} (${minScore} = lowest, ${maxScore} = highest)
66${input.domain ? `**Domain**: ${input.domain}` : ''}
67**Include Examples**: ${input.includeExamples}
68 
69Generate a rubric with:
701. Clear descriptions for each score level
712. Specific characteristics that define each level
723. ${input.includeExamples ? 'Brief example text for each level' : 'No examples needed'}
734. General scoring guidelines
745. Edge cases with guidance
75 
76Respond with valid JSON:
77{
78  "levels": [
79    {
80      "score": ${minScore},
81      "label": "Label (e.g., Poor)",
82      "description": "Detailed description of this level",
83      "characteristics": ["characteristic 1", "characteristic 2"],
84      "example": ${input.includeExamples ? '"Brief example text"' : 'null'}
85    }
86    // ... all levels from ${minScore} to ${maxScore}
87  ],
88  "scoringGuidelines": [
89    "General guideline 1",
90    "General guideline 2"
91  ],
92  "edgeCases": [
93    {
94      "situation": "Edge case description",
95      "guidance": "How to handle it"
96    }
97  ]
98}`;
99 
100  try {
101    const result = await generateText({
102      model: openai(config.openai.model),
103      system: systemPrompt,
104      prompt: userPrompt,
105      temperature: 0.4
106    });
107 
108    const parsed = JSON.parse(result.text);
109 
110    return {
111      success: true,
112      criterion: {
113        name: input.criterionName,
114        description: input.criterionDescription
115      },
116      scale: {
117        min: minScore,
118        max: maxScore,
119        type: input.scale
120      },
121      levels: parsed.levels,
122      scoringGuidelines: parsed.scoringGuidelines,
123      edgeCases: parsed.edgeCases,
124      metadata: {
125        domain: input.domain || null,
126        strictness: input.strictness,
127        generationTimeMs: Date.now() - startTime
128      }
129    };
130  } catch (error) {
131    return {
132      success: false,
133      criterion: {
134        name: input.criterionName,
135        description: input.criterionDescription
136      },
137      scale: {
138        min: minScore,
139        max: maxScore,
140        type: input.scale
141      },
142      levels: [],
143      scoringGuidelines: [],
144      edgeCases: [],
145      metadata: {
146        domain: input.domain || null,
147        strictness: input.strictness,
148        generationTimeMs: Date.now() - startTime
149      }
150    };
151  }
152}
153 
154export const generateRubricTool = tool({
155  description: `Generate a scoring rubric for an evaluation criterion.
156Creates detailed descriptions for each score level.
157Use to establish consistent evaluation standards.`,
158  parameters: GenerateRubricInputSchema,
159  execute: executeGenerateRubric
160});
161 
162