Adding Evaluations

src/agent/qa-agent/
├── agent.ts       # Agent definition
└── eval.ts        # Evals with named exports

import agent from './agent';
 
// Score eval: returns 0-1 quality score
export const confidenceEval = agent.createEval('confidence-check', {
  description: 'Scores output based on confidence level',
  handler: async (ctx, input, output) => {
    const passed = output.confidence >= 0.8;
    return {
      passed,
      score: output.confidence,
      metadata: { threshold: 0.8 },
    };
  },
});

import ragAgent from './agent';
import OpenAI from 'openai';
import { s } from '@agentuity/schema';
 
const client = new OpenAI();
 
const GroundingSchema = s.object({
  isGrounded: s.boolean(),
  unsupportedClaims: s.array(s.string()),
  score: s.number(),
});
 
export const hallucinationEval = ragAgent.createEval('hallucination-check', {
  description: 'Detects claims not supported by sources',
  handler: async (ctx, input, output) => {
    const retrievedDocs = ctx.state.get('retrievedDocs') as string[];
 
    const completion = await client.chat.completions.create({
      model: 'gpt-5-nano',
      response_format: {
        type: 'json_schema',
        json_schema: {
          name: 'grounding_check',
          schema: s.toJSONSchema(GroundingSchema) as Record<string, unknown>,
          strict: true,
        },
      },
      messages: [{
        role: 'user',
        content: `Check if this answer is supported by the source documents.
 
Question: ${input.question}
Answer: ${output.answer}
 
Sources:
${retrievedDocs.join('\n\n')}
 
Identify any claims not supported by the sources.`,
      }],
    });
 
    const result = JSON.parse(completion.choices[0]?.message?.content ?? '{}');
    return {
      passed: result.isGrounded,
      score: result.score,
      reason: result.isGrounded ? 'Answer is grounded in sources' : 'Found unsupported claims',
      metadata: {
        isGrounded: result.isGrounded,
        unsupportedClaims: result.unsupportedClaims,
      },
    };
  },
});

import { createAgent } from '@agentuity/runtime';
import { generateText, generateObject } from 'ai';
import { openai } from '@ai-sdk/openai';
import { z } from 'zod';
 
const ScoresSchema = z.object({
  creativity: z.number().min(0).max(1),
  engagement: z.number().min(0).max(1),
  toneMatch: z.boolean(),
});
 
const agent = createAgent('Story Generator', {
  schema: {
    input: z.object({ prompt: z.string(), tone: z.string() }),
    output: z.object({
      story: z.string(),
      scores: ScoresSchema,
    }),
  },
  handler: async (ctx, input) => {
    // Generate the story
    const { text: story } = await generateText({
      model: openai('gpt-5-mini'),
      prompt: `Write a short ${input.tone} story about: ${input.prompt}`,
    });
 
    // Inline LLM-as-judge: scores returned with response
    const { object: scores } = await generateObject({
      model: openai('gpt-5-nano'),
      schema: ScoresSchema,
      prompt: `Score this ${input.tone} story (0-1 for scores, boolean for tone match):
 
${story}
 
- creativity: How original and imaginative?
- engagement: How compelling to read?
- toneMatch: Does it match the requested "${input.tone}" tone?`,
    });
 
    return { story, scores };  // Frontend receives scores directly
  },
});
 
export default agent;

import agent from './agent';
 
export const externalValidationEval = agent.createEval('external-validation', {
  description: 'Validates output via external API',
  handler: async (ctx, input, output) => {
    try {
      const response = await fetch('https://api.example.com/validate', {
        method: 'POST',
        body: JSON.stringify({ text: output.answer }),
        signal: AbortSignal.timeout(3000),
      });
 
      if (!response.ok) {
        return { success: false, passed: false, error: `Service error: ${response.status}` };
      }
 
      const result = await response.json();
      return { passed: result.isValid };
    } catch (error) {
      ctx.logger.error('Validation failed', { error });
      return { success: false, passed: false, error: error.message };
    }
  },
});

import { politeness } from '@agentuity/evals';
import { openai } from '@ai-sdk/openai';
 
// Override model and threshold
export const politenessCheck = agent.createEval(politeness({
  name: 'politeness-strict',
  model: openai('gpt-5-nano'),
  threshold: 0.9,  // Stricter passing threshold
}));

import agent from './agent';
import { politeness } from '@agentuity/evals';
import type { AgentInput, AgentOutput } from './agent';
 
// Agent schema: { value: number } -> { result: number, doubled: boolean }
// Eval expects: { request: string } -> { response: string }
 
export const politenessCheck = agent.createEval(
  politeness<typeof AgentInput, typeof AgentOutput>({
    middleware: {
      transformInput: (input) => ({
        request: `Calculate double of ${input.value}`,
      }),
      transformOutput: (output) => ({
        response: `Result: ${output.result}, Doubled: ${output.doubled}`,
      }),
    },
  })
);