LLM evaluation in .NET: testing response quality with Microsoft.Extensions.AI.Evaluation

services:
  ollama:
    image: ollama/ollama:latest
    ports:
      - "11434:11434"
    volumes:
      - ollama-models:/root/.ollama
    healthcheck:
      test: ["CMD", "ollama", "list"]
      interval: 10s
      timeout: 5s
      retries: 10

  # pulls llama3.1 into the shared volume, then exits
  ollama-init:
    image: ollama/ollama:latest
    depends_on:
      ollama:
        condition: service_healthy
    environment:
      OLLAMA_HOST: http://ollama:11434
    entrypoint: ["/bin/sh", "-c", "ollama pull llama3.1"]
    restart: "no"

volumes:
  ollama-models:

$ dotnet new sln -n LlmEvalDemo
$ dotnet new classlib -n SupportAssistant -o src/SupportAssistant -f net8.0
$ dotnet new xunit -n SupportAssistant.Evaluation.Tests \
    -o tests/SupportAssistant.Evaluation.Tests -f net8.0
$ dotnet sln add src/SupportAssistant tests/SupportAssistant.Evaluation.Tests

# the class library only sees the abstractions
$ dotnet add src/SupportAssistant package Microsoft.Extensions.AI.Abstractions

# all the evaluation lives in the test project
$ cd tests/SupportAssistant.Evaluation.Tests
$ dotnet add reference ../../src/SupportAssistant
$ dotnet add package Microsoft.Extensions.AI
$ dotnet add package Microsoft.Extensions.AI.Evaluation
$ dotnet add package Microsoft.Extensions.AI.Evaluation.Quality
$ dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting
$ dotnet add package OllamaSharp

# the aieval tool for the report
$ cd ../.. && dotnet new tool-manifest
$ dotnet tool install Microsoft.Extensions.AI.Evaluation.Console

using Microsoft.Extensions.AI;

namespace SupportAssistant;

// The system under test: an assistant that answers ONLY from the
// provided context — exactly the behaviour the GroundednessEvaluator
// will score in the suite.
public class SupportAssistantService(IChatClient chat)
{
    private const string SystemPrompt =
        "You are the support assistant of a software product. " +
        "Answer using ONLY the information in the provided context. " +
        "If the context does not contain the answer, say you don't know " +
        "and suggest contacting support. " +
        "Keep the answer under 80 words and reply in the same language " +
        "as the question.";

    // Returns messages + response: evaluators judge the answer against
    // the whole conversation, not in isolation.
    public async Task<(IList<ChatMessage> Messages, ChatResponse Response)> AskAsync(
        string question,
        string context,
        CancellationToken ct = default)
    {
        IList<ChatMessage> messages =
        [
            new(ChatRole.System, SystemPrompt),
            new(ChatRole.User, $"Context:\n{context}\n\nQuestion: {question}")
        ];

        ChatOptions options = new() { Temperature = 0f };
        ChatResponse response = await chat.GetResponseAsync(messages, options, ct);
        return (messages, response);
    }
}

using Microsoft.Extensions.AI;
using Microsoft.Extensions.AI.Evaluation;
using Microsoft.Extensions.AI.Evaluation.Quality;
using Microsoft.Extensions.AI.Evaluation.Reporting;
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
using Microsoft.Extensions.Configuration;
using OllamaSharp;

namespace SupportAssistant.Evaluation.Tests;

public static class EvalSetup
{
    private static readonly IConfigurationRoot Config =
        new ConfigurationBuilder()
            .AddJsonFile("appsettings.tests.json", optional: false)
            .AddEnvironmentVariables()
            .Build();

    // The assistant under test uses the SUT model...
    public static IChatClient CreateSutClient() =>
        new OllamaApiClient(new Uri(Config["Ollama:Endpoint"]!), Config["Ollama:SutModel"]!);

    // ...the LLM-as-judge evaluators use the judge model: two config
    // entries, so the judge can grow bigger without touching code.
    private static ChatConfiguration CreateJudgeConfiguration() =>
        new(new OllamaApiClient(new Uri(Config["Ollama:Endpoint"]!), Config["Ollama:JudgeModel"]!));

    // One ExecutionName per run: the report groups results by run.
    private static string ExecutionName { get; } = $"{DateTime.Now:yyyyMMddTHHmmss}";

    public static ReportingConfiguration Reporting { get; } =
        DiskBasedReportingConfiguration.Create(
            storageRootPath: FindReportsPath(),   // <repo>/TestReports
            evaluators:
            [
                new RelevanceEvaluator(),
                new CoherenceEvaluator(),
                new GroundednessEvaluator(),
                new AnswerLengthEvaluator()
            ],
            chatConfiguration: CreateJudgeConfiguration(),
            enableResponseCaching: true,
            executionName: ExecutionName);
}

using System.Text.RegularExpressions;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.AI.Evaluation;

namespace SupportAssistant.Evaluation.Tests;

// Deterministic metric, zero LLM: counts words and fails answers over
// budget. The ideal hard gate for CI.
public class AnswerLengthEvaluator : IEvaluator
{
    public const string MetricName = "Answer Length";

    private const int MinWords = 3;
    private const int MaxWords = 100;

    public IReadOnlyCollection<string> EvaluationMetricNames => [MetricName];

    public ValueTask<EvaluationResult> EvaluateAsync(
        IEnumerable<ChatMessage> messages,
        ChatResponse modelResponse,
        ChatConfiguration? chatConfiguration = null,
        IEnumerable<EvaluationContext>? additionalContext = null,
        CancellationToken cancellationToken = default)
    {
        int words = Regex.Matches(modelResponse.Text ?? "", @"\b\w+\b").Count;

        NumericMetric metric = new(
            MetricName,
            value: words,
            reason: $"The response contains {words} words (budget: {MinWords}-{MaxWords}).");

        metric.Interpretation = words >= MinWords && words <= MaxWords
            ? new EvaluationMetricInterpretation(
                EvaluationRating.Good,
                reason: "The response respects the word budget.")
            : new EvaluationMetricInterpretation(
                EvaluationRating.Unacceptable,
                failed: true,
                reason: "The response is empty, too short or over the word budget.");

        return new ValueTask<EvaluationResult>(new EvaluationResult(metric));
    }
}

public record EvalCase(string Name, string Question, string Context)
{
    public override string ToString() => Name;
}

public class SupportAssistantEvalTests(ITestOutputHelper output)
{
    private static bool Strict =>
        Environment.GetEnvironmentVariable("EVAL_STRICT") == "1";

    public static TheoryData<EvalCase> Cases => new()
    {
        new EvalCase("PasswordReset", "How do I reset my password?", AccountFaq),
        new EvalCase("InvoiceDownload", "Dove scarico le fatture in PDF?", BillingFaq),
        new EvalCase("OutOfScope", "Can I pay my invoices with Bitcoin?", BillingFaq)
    };

    [Theory]
    [MemberData(nameof(Cases))]
    public async Task Answer_quality_is_evaluated_and_reported(EvalCase evalCase)
    {
        // await using: results are persisted to disk on dispose.
        await using ScenarioRun scenario =
            await EvalSetup.Reporting.CreateScenarioRunAsync(
                $"SupportAssistant.{evalCase.Name}");

        SupportAssistantService assistant = new(EvalSetup.CreateSutClient());
        (IList<ChatMessage> messages, ChatResponse response) =
            await assistant.AskAsync(evalCase.Question, evalCase.Context);

        // The assistant's context is also the judge's grounding context.
        EvaluationResult result = await scenario.EvaluateAsync(
            messages,
            response,
            additionalContext: [new GroundednessEvaluatorContext(evalCase.Context)]);

        // Deterministic gate: always enforced.
        NumericMetric length = result.Get<NumericMetric>(AnswerLengthEvaluator.MetricName);
        Assert.False(length.Interpretation!.Failed, length.Interpretation.Reason);

        // Judge metrics: always logged, blocking only in strict mode.
        foreach (string name in new[]
        {
            RelevanceEvaluator.RelevanceMetricName,
            CoherenceEvaluator.CoherenceMetricName,
            GroundednessEvaluator.GroundednessMetricName
        })
        {
            // A small judge may fail to produce a parseable score:
            // TryGetValue, so a missing metric is data in the report,
            // not an exception in the suite.
            if (!result.Metrics.TryGetValue(name, out EvaluationMetric? raw)
                || raw is not NumericMetric metric)
            {
                output.WriteLine($"  {name}: no score produced by the judge");
                continue;
            }

            output.WriteLine($"  {name}: {metric.Value} — {metric.Reason}");

            if (Strict)
                Assert.True(
                    metric.Interpretation?.Failed != true,
                    $"{name} failed: {metric.Interpretation?.Reason}");
        }
    }
}

$ git clone https://github.com/fscamuzzi/llm-eval-dotnet-ollama.git
$ cd llm-eval-dotnet-ollama

# dependencies in Docker (Ollama + llama3.1 pull)
$ docker compose up -d
$ docker compose logs -f ollama-init   # wait for the pull

# the evaluation suite
$ dotnet test

# the HTML report
$ dotnet tool restore
$ dotnet tool run aieval report --path TestReports --output report.html