Observability & Monitoring

/// <summary>
/// Creates an OpenTelemetry span for every SCRIPE request handler.
/// Auto-detects the module from the handler's namespace.
/// Pipeline order: Exception → Validation → Auth → FeatureCheck →
///                 Cache → TRACING → Audit → Handler
/// </summary>
public class TracingBehavior<TRequest, TResponse>
    : IPipelineBehavior<TRequest, TResponse>
{
    private static readonly ActivitySource Source = new("SCRIPE.Mediator");

    public async Task<TResponse> Handle(
        TRequest request, RequestHandlerDelegate<TResponse> next,
        CancellationToken ct)
    {
        var requestName = typeof(TRequest).Name;
        var kind = requestName.EndsWith("Query")
            ? "Query" : "Command";
        var module = DetectModule(typeof(TRequest).Namespace);

        using var activity = Source.StartActivity(
            $"AstraFlow mediator {kind}: {requestName}",
            ActivityKind.Internal);

        activity?.SetTag("mediatr.request_type", requestName);
        activity?.SetTag("mediatr.kind", kind);
        activity?.SetTag("mediatr.module", module);

        try
        {
            var response = await next();
            activity?.SetStatus(ActivityStatusCode.Ok);
            return response;
        }
        catch (Exception ex)
        {
            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
            activity?.RecordException(ex);
            throw;
        }
    }
}

global:
  scrape_interval: 15s
  evaluation_interval: 15s

# Load alert rules
rule_files:
  - /etc/prometheus/alerts/*.yml

scrape_configs:
  # ── Monolith Mode (single target) ──
  - job_name: 'scripe-backend'
    static_configs:
      - targets: ['host.docker.internal:5035']
    metrics_path: '/metrics'
    scrape_interval: 15s

  # ── Microservice Mode (one target per module) ──
  # Uncomment and configure per deployed module:
  # - job_name: 'scripe-identity'
  #   static_configs:
  #     - targets: ['identity-service:5001']
  # - job_name: 'scripe-entitlements'
  #   static_configs:
  #     - targets: ['entitlements-service:5004']
  # - job_name: 'scripe-gateway'
  #   static_configs:
  #     - targets: ['gateway-service:5010']

builder.Host.UseSerilog((context, loggerConfig) =>
{
    loggerConfig
        .ReadFrom.Configuration(context.Configuration)
        .Enrich.FromLogContext()
        .Enrich.WithMachineName()
        .Enrich.WithEnvironmentName()
        .Enrich.WithProperty("Application", "SCRIPE");

    // Loki sink — opt-in via configuration
    // Leave Loki:Url empty in dev to disable (logs go to console only)
    // Set Loki:Url in staging/production to enable centralized logging
    var lokiUrl = context.Configuration["Loki:Url"];
    if (!string.IsNullOrEmpty(lokiUrl))
    {
        loggerConfig.WriteTo.GrafanaLoki(lokiUrl,
            labels: [
                new LokiLabel { Key = "app", Value = "scripe" },
                new LokiLabel { Key = "environment",
                    Value = context.HostingEnvironment.EnvironmentName },
            ],
            propertiesAsLabels: [
                "Application", "CorrelationId",
                "TenantId", "ModuleTag"
            ]);
    }
});

# Start the full monitoring stack (from SCRIPE root directory)
docker compose -f infrastructure/monitoring/docker-compose.monitoring.yml up -d

# ═══════════════════════════════════════════════════════════
# Access Points:
# ═══════════════════════════════════════════════════════════
# Grafana:    http://localhost:3001  (admin / scripe-admin)
# Prometheus: http://localhost:9090  (metrics store + alerts)
# Jaeger:     http://localhost:16686 (distributed tracing UI)
# Loki:       http://localhost:3100  (log aggregation API only)
#
# Your app exposes:
#   /metrics  → Prometheus scrapes this every 15s
#   OTLP :4317 → Jaeger receives traces via gRPC
#   Serilog   → Pushes logs to Loki (when Loki:Url is set)
# ═══════════════════════════════════════════════════════════

# Verify everything is running
docker ps --filter name=scripe- --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

# View Prometheus targets (should show "UP")
curl http://localhost:9090/api/v1/targets | jq '.data.activeTargets[].health'

# Stop the monitoring stack
docker compose -f infrastructure/monitoring/docker-compose.monitoring.yml down

{
  "Observability": {
    "ExporterType": "Console",
    "ServiceName": "SCRIPE-Dev",
    "OtlpEndpoint": "http://localhost:4317",
    "TraceSampleRatio": 1.0,
    "EnablePrometheus": true,
    "Enabled": true
  },
  "Loki": {
    "Url": ""
  }
}

{
  "Observability": {
    "ExporterType": "Otlp",
    "ServiceName": "SCRIPE-Production",
    "OtlpEndpoint": "http://jaeger:4317",
    "TraceSampleRatio": 0.1,
    "EnablePrometheus": true,
    "Enabled": true
  },
  "Loki": {
    "Url": "http://loki:3100"
  }
}