##
## Prometheus ConfigMap for Fogbreak AI monitoring.
## Scrapes vLLM endpoints and FastAPI proxy metrics.
##
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: fogbreak-ai
  labels:
    app: fogbreak
    component: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s

    rule_files:
      - /etc/prometheus/rules/*.yml

    alerting:
      alertmanagers:
        - static_configs:
            - targets:
                - alertmanager:9093

    scrape_configs:
      # vLLM Llama endpoints
      - job_name: vllm-llama
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names:
                - fogbreak-ai
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_model]
            regex: llama-reasoning
            action: keep
          - source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: ${1}:8000

      # vLLM Mistral endpoints
      - job_name: vllm-mistral
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names:
                - fogbreak-ai
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_model]
            regex: mistral-fast
            action: keep
          - source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: ${1}:8001

      # FastAPI AI Proxy
      - job_name: fogbreak-ai-proxy
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names:
                - fogbreak-ai
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_component]
            regex: ai-proxy
            action: keep
          - source_labels: [__meta_kubernetes_pod_ip]
            target_label: __address__
            replacement: ${1}:8080
          - target_label: __metrics_path__
            replacement: /metrics

      # GPU metrics via DCGM exporter
      - job_name: dcgm-exporter
        kubernetes_sd_configs:
          - role: pod
            namespaces:
              names:
                - fogbreak-ai
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_label_app]
            regex: dcgm-exporter
            action: keep

  alerts.yml: |
    groups:
      - name: fogbreak-ai-alerts
        rules:
          # High inference latency
          - alert: HighInferenceLatency
            expr: fogbreak_ai_endpoint_latency_ms > 5000
            for: 5m
            labels:
              severity: warning
              team: platform
            annotations:
              summary: "High AI inference latency on {{ $labels.endpoint }}"
              description: "Average latency {{ $value }}ms exceeds 5s threshold for 5 minutes."

          # Endpoint down
          - alert: VLLMEndpointDown
            expr: fogbreak_ai_endpoint_healthy == 0
            for: 2m
            labels:
              severity: critical
              team: platform
            annotations:
              summary: "vLLM endpoint {{ $labels.endpoint }} is down"
              description: "Endpoint has been unhealthy for 2+ minutes. Model: {{ $labels.model }}"

          # High error rate
          - alert: HighErrorRate
            expr: rate(fogbreak_ai_errors_total[5m]) / rate(fogbreak_ai_requests_total[5m]) > 0.05
            for: 5m
            labels:
              severity: warning
              team: platform
            annotations:
              summary: "AI inference error rate above 5%"
              description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."

          # GPU memory pressure
          - alert: GPUMemoryHigh
            expr: DCGM_FI_DEV_FB_USED / DCGM_FI_DEV_FB_TOTAL > 0.95
            for: 5m
            labels:
              severity: critical
              team: platform
            annotations:
              summary: "GPU {{ $labels.gpu }} memory usage above 95%"
              description: "GPU memory at {{ $value | humanizePercentage }}. Risk of OOM."

          # No requests (model may be wasting GPU)
          - alert: IdleGPU
            expr: rate(fogbreak_ai_endpoint_requests[1h]) == 0
            for: 2h
            labels:
              severity: info
              team: platform
            annotations:
              summary: "GPU endpoint {{ $labels.endpoint }} idle for 2+ hours"
              description: "Consider scaling down to save GPU costs."

          # All endpoints down
          - alert: AllEndpointsDown
            expr: sum(fogbreak_ai_endpoint_healthy) == 0
            for: 1m
            labels:
              severity: critical
              team: platform
            annotations:
              summary: "All Fogbreak AI endpoints are down"
              description: "Complete AI inference outage. No healthy endpoints available."
