{
  "uid": "docs-copilot",
  "title": "Docs copilot",
  "tags": [
    "llm",
    "ai-engineering"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "templating": {
    "list": [
      {
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus",
        "label": "Prometheus",
        "current": {},
        "hide": 0
      }
    ]
  },
  "annotations": {
    "list": []
  },
  "description": "Traffic, latency, tokens, spend, grounded rate and guardrails for the docs copilot (AI Engineering course, lesson 21). Metric names assume Prometheus's default OTLP translation.",
  "panels": [
    {
      "id": 1,
      "type": "timeseries",
      "title": "Requests / min",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqpm"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "60 * sum(rate(app_request_duration_seconds_count[5m]))",
          "legendFormat": "requests/min"
        }
      ]
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "p95 request latency",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum by (le) (rate(app_request_duration_seconds_bucket[5m])))",
          "legendFormat": "p95"
        }
      ]
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "p95 model latency by model",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket[5m])))",
          "legendFormat": "{{gen_ai_request_model}}"
        }
      ]
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "Model errors / s by type",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (error_type) (rate(gen_ai_client_operation_duration_seconds_count{error_type!=\"\"}[5m]))",
          "legendFormat": "{{error_type}}"
        }
      ]
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Tokens / min by type",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 16
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "60 * sum by (gen_ai_token_type) (rate(gen_ai_client_token_usage_sum[5m]))",
          "legendFormat": "{{gen_ai_token_type}}"
        }
      ]
    },
    {
      "id": 6,
      "type": "stat",
      "title": "Spend, last hour (USD)",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 16
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(increase(app_cost_usd_total[1h]))",
          "legendFormat": "USD"
        }
      ]
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Grounded answer rate",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 24
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(app_answers_total{grounded=\"true\"}[15m])) / sum(rate(app_answers_total[15m]))",
          "legendFormat": "grounded"
        }
      ]
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Guardrail hits, last hour",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 24
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (type) (increase(app_guardrail_hits_total[1h]))",
          "legendFormat": "{{type}}"
        }
      ]
    }
  ]
}