LLM observability in .NET: traces, tokens and cost with OpenTelemetry and the Aspire dashboard

span name       chat llama3.2:3b
duration        2.41s

gen_ai.operation.name        chat
gen_ai.provider.name         ollama
gen_ai.request.model         llama3.2:3b
gen_ai.response.model        llama3.2:3b
gen_ai.usage.input_tokens    412
gen_ai.usage.output_tokens   128

# added by the middleware I write further down
llmobs.usage.cost_usd        0.000139

services:
  ollama:
    image: ollama/ollama:latest
    ports: ["11434:11434"]
    volumes: [ollama-data:/root/.ollama]
    healthcheck:
      test: ["CMD-SHELL", "ollama list >/dev/null 2>&1 || exit 1"]
      interval: 10s
      retries: 12
      start_period: 20s

  # one-shot: pulls the model as soon as Ollama is healthy, then exits
  ollama-init:
    image: ollama/ollama:latest
    depends_on:
      ollama:
        condition: service_healthy
    environment:
      OLLAMA_HOST: http://ollama:11434
    entrypoint: ["/bin/sh", "-c", "ollama pull ${OLLAMA_MODEL:-llama3.2:3b}"]
    restart: "no"

  aspire-dashboard:
    image: mcr.microsoft.com/dotnet/aspire-dashboard:9.0
    ports:
      - "18888:18888"   # UI
      - "4317:18889"    # OTLP/gRPC
    environment:
      DOTNET_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS: "true"
      ASPNETCORE_URLS: http://+:18888

volumes:
  ollama-data:

dotnet add package Microsoft.Extensions.AI
dotnet add package OllamaSharp

dotnet add package OpenTelemetry.Extensions.Hosting
dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol
dotnet add package OpenTelemetry.Instrumentation.AspNetCore
dotnet add package OpenTelemetry.Instrumentation.Http
dotnet add package OpenTelemetry.Instrumentation.Runtime

namespace LlmObservability.Api.Observability;

// Single source of truth for the names the app emits: this is what keeps
// UseOpenTelemetry() and AddSource()/AddMeter() in sync.
public static class TelemetryNames
{
    // ActivitySource + Meter passed explicitly to UseOpenTelemetry().
    public const string ChatSource = "LlmObservability.Ai";

    // Meter of the cost middleware (custom, outside the semantic conventions).
    public const string CostMeter = "LlmObservability.Cost";

    // ActivitySource of the app, for spans opened by hand.
    public const string AppSource = "LlmObservability.Api";

    // Name Microsoft.Extensions.AI uses when no sourceName is supplied.
    public const string LibraryDefaultSource = "Experimental.Microsoft.Extensions.AI";
}

// Registers the whole OpenTelemetry stack plus the objects the AI pipeline
// needs (price list, calculator, cost Meter).
public static IHostApplicationBuilder AddLlmObservability(this IHostApplicationBuilder builder)
{
    // 1) Price list and calculator, shared with the cost middleware.
    builder.Services.Configure<PricingOptions>(builder.Configuration.GetSection(PricingOptions.SectionName));
    builder.Services.AddSingleton(sp =>
        new CostCalculator(sp.GetRequiredService<IOptions<PricingOptions>>().Value));

    // 2) The Meter carrying the custom counter: singleton, stable name.
    builder.Services.AddSingleton(_ => new Meter(TelemetryNames.CostMeter));

    // 3) Who is emitting: service.name is the field everything groups by.
    var serviceName = builder.Configuration["Otel:ServiceName"] ?? "llm-observability-api";
    var otlpEndpoint = builder.Configuration["Otel:Endpoint"] ?? "http://localhost:4317";

    builder.Services.AddOpenTelemetry()
        .ConfigureResource(r => r.AddService(serviceName))
        .WithTracing(tracing => tracing
            .AddAspNetCoreInstrumentation()   // HTTP root span
            .AddHttpClientInstrumentation()   // the real call to Ollama
            .AddSource(TelemetryNames.ChatSource)
            .AddSource(TelemetryNames.LibraryDefaultSource)
            .AddSource(TelemetryNames.AppSource)
            .AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)))
        .WithMetrics(metrics => metrics
            .AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddRuntimeInstrumentation()
            .AddMeter(TelemetryNames.ChatSource)          // token/duration histograms
            .AddMeter(TelemetryNames.LibraryDefaultSource)
            .AddMeter(TelemetryNames.CostMeter)           // the cost counter
            .AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint)));

    // 4) Logs on the same pipe: warnings and spans correlated by trace id.
    builder.Logging.AddOpenTelemetry(logging =>
    {
        logging.IncludeFormattedMessage = true;
        logging.IncludeScopes = true;
        logging.AddOtlpExporter(o => o.Endpoint = new Uri(otlpEndpoint));
    });

    return builder;
}

// ChatClientBuilder wraps in reverse registration order: the FIRST stage
// registered is the OUTERMOST. OpenTelemetry therefore measures the whole
// call, tool round-trips included, and the cost stage runs inside its span.
public static IHostApplicationBuilder AddInstrumentedChatClient(this IHostApplicationBuilder builder)
{
    builder.Services.Configure<OllamaOptions>(builder.Configuration.GetSection(OllamaOptions.SectionName));

    // Prompts and completions on spans: gold in development, a GDPR incident in production.
    var enableSensitiveData = builder.Configuration.GetValue("Otel:EnableSensitiveData", false);

    builder.Services.AddChatClient(sp =>
    {
        var options = sp.GetRequiredService<IOptions<OllamaOptions>>().Value;
        return new OllamaApiClient(new Uri(options.Endpoint), options.Model);
    })
        // Outermost: span and histograms cover everything below.
        .UseOpenTelemetry(
            sourceName: TelemetryNames.ChatSource,
            configure: c => c.EnableSensitiveData = enableSensitiveData)
        // Then cost, so its tags land on the span opened above.
        .Use((inner, sp) => new CostTrackingChatClient(
            inner,
            sp.GetRequiredService<CostCalculator>(),
            sp.GetRequiredService<Meter>(),
            sp.GetRequiredService<ILogger<CostTrackingChatClient>>()))
        // Innermost: the tool loop stays inside the span.
        .UseFunctionInvocation();

    return builder;
}

// Stage that turns the tokens reported by the model into money. It sits INSIDE
// UseOpenTelemetry(), so its tags land on the gen_ai span already opened.
public sealed class CostTrackingChatClient : DelegatingChatClient
{
    private const string CostAttribute = "llmobs.usage.cost_usd";

    // Blocking call: delegate, then price whatever usage came back.
    public override async Task<ChatResponse> GetResponseAsync(
        IEnumerable<ChatMessage> messages,
        ChatOptions? options = null,
        CancellationToken cancellationToken = default)
    {
        var response = await base.GetResponseAsync(messages, options, cancellationToken);
        Record(response.ModelId, response.Usage);
        return response;
    }

    // When streaming, usage arrives as UsageContent inside the updates:
    // accumulate and price once, when the stream ends.
    public override async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
        IEnumerable<ChatMessage> messages,
        ChatOptions? options = null,
        [EnumeratorCancellation] CancellationToken cancellationToken = default)
    {
        var totals = new UsageDetails();
        string? modelId = null;

        await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken))
        {
            modelId ??= update.ModelId;
            var usages = update.Contents.OfType<UsageContent>().Select(c => c.Details);
            usages.ToList().ForEach(totals.Add);
            yield return update;
        }

        Record(modelId, totals);
    }

    // Computes the cost, records it on the counter and tags the current span.
    private void Record(string? modelId, UsageDetails? usage)
    {
        // 1) Guard: some providers omit usage entirely — nothing to price.
        if (usage is null)
            return;

        var model = modelId ?? "unknown";
        var input = usage.InputTokenCount ?? 0;
        var output = usage.OutputTokenCount ?? 0;

        // 2) Which price list values these tokens (shadow pricing).
        var billingModel = _calculator.ResolveBillingModel(model);
        var cost = _calculator.Estimate(billingModel, input, output);

        // 3) A chart flat at zero is a configuration bug far more often than good news.
        if (cost == 0m && (input > 0 || output > 0))
            _logger.LogWarning("No price list entry for model {BillingModel}: cost reported as 0.", billingModel);

        // 4) Counter with the tags you will group by in the dashboard.
        _costCounter.Add((double)cost,
            new KeyValuePair<string, object?>("gen_ai.request.model", model),
            new KeyValuePair<string, object?>("llmobs.billing.model", billingModel));

        // 5) Same number on the span: a slow request also tells you what it cost.
        Activity.Current?.SetTag(CostAttribute, (double)cost);
    }
}

// Turns a token count into a monetary estimate using the configured price list.
// Pure and dependency-free on purpose: this is the piece worth unit-testing.
public sealed class CostCalculator(PricingOptions pricing)
{
    // Model whose price list values the tokens: the configured billing model
    // when set (shadow pricing), otherwise the model that answered.
    public string ResolveBillingModel(string responseModel)
    {
        if (!string.IsNullOrWhiteSpace(pricing.BillingModel))
            return pricing.BillingModel;
        return responseModel;
    }

    // Cost in USD of an input/output token pair for the given model.
    // Returns 0 when the model is not in the price list: typical for a local model.
    public decimal Estimate(string model, long inputTokens, long outputTokens)
    {
        // 1) Guard: an unknown model has no price, so no cost to report.
        if (!pricing.Models.TryGetValue(model, out var price))
            return 0m;

        // 2) Prices are per million tokens, so scale both sides down.
        var input = inputTokens / 1_000_000m * price.InputPerMillionUsd;
        var output = outputTokens / 1_000_000m * price.OutputPerMillionUsd;
        return input + output;
    }
}

// Blocking call: returns the numbers that also land in the dashboard,
// plus the trace id to look the request up by.
private static async Task<IResult> HandleChatAsync(
    ChatRequest request,
    IChatClient chatClient,
    CostCalculator calculator,
    IOptions<PricingOptions> pricing,
    CancellationToken cancellationToken)
{
    // 1) Guard: an empty prompt is a client bug, not a model problem.
    if (string.IsNullOrWhiteSpace(request.Prompt))
        return TypedResults.BadRequest(new ErrorResponse("Prompt must not be empty."));

    // 2) Ask the model, letting the pipeline handle tools and telemetry.
    var messages = BuildMessages(request.Prompt);
    var response = await chatClient.GetResponseAsync(
        messages,
        new ChatOptions { Tools = SupportTools.BuildTools() },
        cancellationToken);

    // 3) Re-derive the same cost the middleware recorded, to expose it.
    var model = response.ModelId ?? "unknown";
    var input = response.Usage?.InputTokenCount ?? 0;
    var output = response.Usage?.OutputTokenCount ?? 0;
    var cost = calculator.Estimate(calculator.ResolveBillingModel(model), input, output);

    return TypedResults.Ok(new ChatReply(
        response.Text,
        model,
        input,
        output,
        cost,
        Activity.Current?.TraceId.ToString() ?? string.Empty));
}

curl -s http://localhost:5173/chat \
  -H 'content-type: application/json' \
  -d '{"prompt":"What is the status of order A-1001?"}'

{
  "text": "Order A-1001 has shipped, delivery expected on 12 August 2026 with BRT.",
  "model": "llama3.2:3b",
  "inputTokens": 383,
  "outputTokens": 75,
  "estimatedCostUsd": 0.00010245,
  "traceId": "d6b5f062116d0af3fc41d7ac617dc81a"
}

{
  "Ollama": {
    "Endpoint": "http://localhost:11434",
    "Model": "llama3.2:3b"
  },
  "Otel": {
    "ServiceName": "llm-observability-api",
    "Endpoint": "http://localhost:4317",
    "EnableSensitiveData": true
  },
  "Pricing": {
    "BillingModel": "gpt-4o-mini",
    "Models": {
      "gpt-4o-mini": { "InputPerMillionUsd": 0.15, "OutputPerMillionUsd": 0.60 }
    }
  }
}

[Fact]
public async Task Pipeline_EmitsAGenAiSpanOnTheConfiguredSource()
{
    // 1) Listen only to the source name the app registers with AddSource().
    var captured = new List<Activity>();
    using var listener = new ActivityListener
    {
        ShouldListenTo = source => source.Name == TelemetryNames.ChatSource,
        Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
        ActivityStopped = captured.Add,
    };
    ActivitySource.AddActivityListener(listener);

    // 2) One call through the pipeline, over a fake provider.
    var (client, meter) = BuildPipeline(billingModel: "gpt-4o-mini");
    using var _ = meter;
    await client.GetResponseAsync("hello");

    // 3) The span must exist and carry the semantic-convention attributes.
    var span = Assert.Single(captured);
    Assert.Contains("gen_ai.usage.input_tokens", span.TagObjects.Select(t => t.Key));
    Assert.Contains("gen_ai.usage.output_tokens", span.TagObjects.Select(t => t.Key));

    // 4) …and my middleware's tag: proof the cost stage runs INSIDE the span.
    Assert.Contains("llmobs.usage.cost_usd", span.TagObjects.Select(t => t.Key));
}

git clone https://github.com/fscamuzzi/llm-observability-dotnet-opentelemetry
cd llm-observability-dotnet-opentelemetry

docker compose up -d
docker compose logs -f ollama-init   # wait for 'success'

dotnet test
dotnet run --project src/LlmObservability.Api